From 8c2c6fab24eb0638a277ad088f0c7ce3cd067459 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 23 Mar 2020 16:16:48 -0500 Subject: [PATCH 001/655] Added base object store support (upload file to cloud, get file from cloud, display file size) --- pom.xml | 33 +++ .../irida/model/project/ReferenceFile.java | 5 +- .../model/sequenceFile/SequenceFile.java | 16 +- .../workflow/analysis/AnalysisOutputFile.java | 5 +- .../processing/impl/FastqcFileProcessor.java | 6 +- .../FilesystemSupplementedRepositoryImpl.java | 40 ++-- .../web/components/IridaFileProperties.java | 101 +++++++++ .../service/IridaFileStorageService.java | 72 +++++++ .../impl/IridaFileStorageServiceImpl.java | 199 ++++++++++++++++++ 9 files changed, 447 insertions(+), 30 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/IridaFileProperties.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/service/IridaFileStorageService.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java diff --git a/pom.xml b/pom.xml index 4a4567fbd76..aa7a9bcc9d3 100644 --- a/pom.xml +++ b/pom.xml @@ -344,6 +344,11 @@ jackson-databind ${jackson.version} + + com.fasterxml.jackson.core + jackson-core + 2.10.3 + com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -667,6 +672,34 @@ jvm-breakglass ${jvm-breakglass.version} + + + com.azure + azure-storage-blob + 12.0.0 + + + jakarta.xml.bind + jakarta.xml.bind-api + + + jakarta.activation + jakarta.activation-api + + + + + + com.amazonaws + aws-java-sdk-s3 + 1.11.376 + + + commons-logging + commons-logging + + + diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index f88a1d57a15..911b7724445 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.MutableIridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; /** * A reference file to be associated with a {@link Project}. @@ -38,6 +39,8 @@ @EntityListeners({AuditingEntityListener.class, RelativePathTranslatorListener.class}) public class ReferenceFile implements VersionedFileFields, MutableIridaThing { + private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -91,7 +94,7 @@ public ReferenceFile(Path file) { @Override public String getLabel() { - return file.getFileName().toString(); + return fileService.getFileName(file); } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index ce708200d5c..519a7a31beb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -1,8 +1,5 @@ package ca.corefacility.bioinformatics.irida.model.sequenceFile; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.Date; import java.util.HashMap; @@ -46,6 +43,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -107,6 +105,8 @@ public class SequenceFile extends IridaResourceSupport implements MutableIridaTh @JoinColumn(name = "remote_status") private RemoteStatus remoteStatus; + private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); + public SequenceFile() { createdDate = new Date(); fileRevisionNumber = 0L; @@ -158,7 +158,7 @@ public void setFile(Path file) { @Override public String getLabel() { - return file.getFileName().toString(); + return fileService.getFileName(file); } @Override @@ -227,13 +227,7 @@ public String getOptionalProperty(String key) { @JsonIgnore public String getFileSize() { String size = "N/A"; - try { - size = IridaSequenceFile.humanReadableByteCount(Files.size(file), true); - } catch (NoSuchFileException e) { - logger.error("Could not find file " + file); - } catch (IOException e) { - logger.error("Could not calculate file size: ", e); - } + size = IridaSequenceFile.humanReadableByteCount(fileService.getFileSize(file), true); return size; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 0bb0bcac3bb..7556305c395 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; /** * Store file references to files produced by a workflow execution that we @@ -64,6 +65,8 @@ public class AnalysisOutputFile extends IridaResourceSupport implements IridaThi @Column(name = "label_prefix") private final String labelPrefix; + private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); + /** * for hibernate */ @@ -130,7 +133,7 @@ public void incrementFileRevisionNumber() { @Override public String getLabel() { - return Strings.isNullOrEmpty(labelPrefix) ? file.toFile().getName() : labelPrefix + '-' + file.toFile().getName(); + return Strings.isNullOrEmpty(labelPrefix) ? fileService.getFileName(file) : labelPrefix + '-' + fileService.getFileName(file); } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 28441d0a619..af25dc23382 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -10,6 +10,8 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -52,6 +54,8 @@ public class FastqcFileProcessor implements FileProcessor { private final AnalysisOutputFileRepository outputFileRepository; private final MessageSource messageSource; + private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); + /** * Create a new {@link FastqcFileProcessor} * @@ -91,7 +95,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx LocaleContextHolder.getLocale())); try { uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - fileToProcess.toFile()); + fileService.getTemporaryFile(fileToProcess)); BasicStats basicStats = new BasicStats(); PerBaseQualityScores pbqs = new PerBaseQualityScores(); PerSequenceQualityScores psqs = new PerSequenceQualityScores(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index 5b66f93e74d..c2b1702cb97 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -20,12 +20,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.ReflectionUtils; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; /** * Custom implementation of a repository that writes the {@link Path} part of an @@ -41,6 +44,9 @@ public abstract class FilesystemSupplementedRepositoryImpl Date: Tue, 24 Mar 2020 14:05:22 -0500 Subject: [PATCH 002/655] Changed to service. Added comments --- .../model/sequenceFile/SequenceFile.java | 1 + .../web/components/IridaFileProperties.java | 101 ------------------ .../impl/IridaFileStorageServiceImpl.java | 20 ++-- 3 files changed, 12 insertions(+), 110 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/IridaFileProperties.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 519a7a31beb..abc0c2e1fd2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -29,6 +29,7 @@ import org.hibernate.envers.NotAudited; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/IridaFileProperties.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/IridaFileProperties.java deleted file mode 100644 index 945bce02a04..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/IridaFileProperties.java +++ /dev/null @@ -1,101 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobProperties; -import com.azure.storage.blob.models.BlobStorageException; - -@Component -public class IridaFileProperties { - - @Value("${irida.storage.type}") - private String storageType; - - @Value("${azure.container.name}") - private String containerName; - - //@Value("${azure.account.connection.string}") - private String connectStr = "DefaultEndpointsProtocol=https;AccountName=stirida;AccountKey=r8ruK3dz7eLevFUqc9bhb8L/KAS0dphfwZQEV3oeOu7+tJmdRCSKyJvtsU+FbfcFrjtLCi/LpNl2nYAG+SWdLQ==;EndpointSuffix=core.windows.net"; - - private static final Logger logger = LoggerFactory.getLogger(SequenceFile.class); - - /** - * This method will get the size in bytes of the file - * either from local storage or a cloud based storage - * - * @return File size in bytes - */ - public Long getFileSize(Path file) { - Long fileSize = 0L; - if(storageType.equalsIgnoreCase("azure")) { - fileSize = getFileSizeFromAzureBlobStorage(file); - } else if (storageType.equalsIgnoreCase("aws")) { - fileSize = getFileSizeFromAwsBucket(); - } else { - try { - fileSize = Files.size(file); - } catch (NoSuchFileException e) { - logger.error("Could not find file " + file); - } catch (IOException e) { - logger.error("Could not calculate file size: ", e); - } - } - return fileSize; - } - - /** - * This method will get the size in bytes of the file - * from azure - * - * @return File size in bytes - */ - private Long getFileSizeFromAzureBlobStorage(Path file) { - Long fileSize = 0L; - // Create a BlobServiceClient object - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(connectStr) - .buildClient(); - BlobContainerClient containerClient = blobServiceClient.getBlobContainerClient(containerName); - BlobClient blobClient; - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString().substring(1)); - - try { - BlobProperties properties = blobClient.getProperties(); - logger.debug("Size of blob:" + Long.toString(properties.getBlobSize())); - fileSize = properties.getBlobSize(); - } catch(BlobStorageException e) { - logger.debug("File not found: " + e); - } - return fileSize; - } - - /** - * This method will get the size in bytes of the file - * from aws - * - * @return File size in bytes - */ - private Long getFileSizeFromAwsBucket() { - Long fileSize = 0L; - - // Implement code to get file size from AWS - - return fileSize; - } - -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java index abceb9803c1..870c0a96213 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -19,12 +20,10 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobProperties; import com.azure.storage.blob.models.BlobStorageException; @Service public class IridaFileStorageServiceImpl implements IridaFileStorageService { - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageServiceImpl.class); @Value("${irida.storage.type}") @@ -36,11 +35,11 @@ public class IridaFileStorageServiceImpl implements IridaFileStorageService { @Value("${azure.account.connection.string}") private String connectStr; - // Create an Azure BlobServiceClient object private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; private BlobClient blobClient; + @Autowired public IridaFileStorageServiceImpl(){ this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectStr) .buildClient(); @@ -89,10 +88,10 @@ public File getTemporaryFile(Path file) { public Long getFileSize(Path file) { Long fileSize = 0L; if(storageType.equalsIgnoreCase("azure")) { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString().substring(1)); try { + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(file.toAbsolutePath() + .toString().substring(1)); fileSize = blobClient.getProperties().getBlobSize(); } catch (BlobStorageException e) { logger.debug("Couldn't calculate size as the file was not found [" + e + "]"); @@ -126,10 +125,8 @@ public void writeFile(Path source, Path target) { logger.debug("Uploading file to azure: [" + target.getFileName() + "]"); blobClient.uploadFromFile(source.toString(), false); logger.debug("File uploaded to: [" + blobClient.getBlobUrl() + "]"); - - BlobProperties properties = blobClient.getProperties(); - logger.debug("Size of blob:" + Long.toString(properties.getBlobSize())); } else if(storageType.equalsIgnoreCase("aws")){ + //implement aws code to upload file to s3 bucket } else { try { Files.move(source, target); @@ -179,11 +176,15 @@ public boolean storageTypeIsLocal(){ public String getFileName(Path file) { String fileName = ""; + logger.debug("STORAGE TYPE IS: " + storageType); + if(storageType.equalsIgnoreCase("azure")) { blobClient = containerClient.getBlobClient(file.toAbsolutePath() .toString() .substring(1)); try { + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. String[] blobNameTokens = blobClient.getBlobName() .split("/"); fileName = blobNameTokens[blobNameTokens.length - 1]; @@ -191,6 +192,7 @@ public String getFileName(Path file) { logger.debug("Couldn't find file [" + e + "]"); } } else if(storageType.equalsIgnoreCase("aws")) { + //implement aws code to get file name from aws s3 bucket } else { fileName = file.toFile().getName(); } From c80a6c5bb054066f6252564ced100da538162c0c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 27 Mar 2020 19:18:10 -0500 Subject: [PATCH 003/655] Created subclasses of sequence file to support local and cloud based storage. Updated creating of sequence files. Updated tests --- .../services/IridaApiServicesConfig.java | 19 +++++++ .../irida/model/project/ReferenceFile.java | 6 +-- .../model/sequenceFile/CloudSequenceFile.java | 48 +++++++++++++++++ .../model/sequenceFile/LocalSequenceFile.java | 51 +++++++++++++++++++ .../model/sequenceFile/SequenceFile.java | 23 +-------- .../workflow/analysis/AnalysisOutputFile.java | 6 +-- .../impl/SequenceFilePairConcatenator.java | 12 +++-- .../SingleEndSequenceFileConcatenator.java | 8 ++- .../processing/impl/FastqcFileProcessor.java | 7 ++- .../FilesystemSupplementedRepositoryImpl.java | 2 - .../IridaFileStorageServiceImpl.java | 25 +++++---- .../ria/web/samples/SamplesController.java | 6 ++- .../impl/IridaFileStorageFactoryImpl.java | 43 ++++++++++++++++ .../impl/SequencingObjectServiceImpl.java | 2 +- .../sequencefile/SequenceFileResource.java | 8 ++- .../RESTSampleSequenceFilesController.java | 6 ++- .../irida/events/ProjectEventHandlerTest.java | 7 +-- .../unit/SequenceFilePairTest.java | 9 ++-- .../unit/AnalysisSubmissionTest.java | 3 +- .../SequenceFilePairConcatenatorTest.java | 6 ++- ...SingleEndSequenceFileConcatenatorTest.java | 3 +- .../AutomatedAnalysisFileProcessorTest.java | 14 ++--- .../impl/unit/ChecksumFileProcessorTest.java | 5 +- .../impl/unit/CoverageFileProcessorTest.java | 7 +-- .../impl/unit/FastqcFileProcessorTest.java | 9 ++-- .../impl/unit/GzipFileProcessorTest.java | 8 +-- .../SequenceFileRepositoryImplTest.java | 13 ++--- .../irida/ria/unit/TestDataFactory.java | 5 +- .../unit/web/SequencingRunControllerTest.java | 5 +- .../web/files/SequenceFileControllerTest.java | 3 +- .../ProjectSamplesControllerTest.java | 6 +-- .../web/samples/SamplesControllerTest.java | 15 +++--- .../ReadAnalysisSubmissionPermissionTest.java | 7 +-- .../service/DatabaseSetupGalaxyITService.java | 13 +++-- .../AnalysisExecutionServiceGalaxyTest.java | 4 +- .../AnalysisWorkspaceServiceGalaxyIT.java | 7 +-- .../AnalysisWorkspaceServiceGalaxyTest.java | 11 ++-- .../export/ExportUploadServiceTest.java | 3 +- .../NcbiExportSubmissionServiceTest.java | 5 +- .../SequencingObjectServiceImplIT.java | 21 +++----- .../SequencingRunServiceImplIT.java | 6 ++- .../unit/sample/SampleServiceImplTest.java | 17 ++++--- ...eEndSequenceFileRemoteServiceImplTest.java | 4 +- .../controller/test/unit/TestDataFactory.java | 5 +- 44 files changed, 330 insertions(+), 163 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java rename src/main/java/ca/corefacility/bioinformatics/irida/{service/impl => repositories/filesystem}/IridaFileStorageServiceImpl.java (92%) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 3d7372299a7..d97135cb03f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -16,6 +16,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.impl.*; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionCleanupService; @@ -24,6 +25,7 @@ import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.AnalysisSubmissionCleanupServiceImpl; import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.util.IridaPluginMessageSource; + import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import net.matlux.NreplServerSpring; @@ -125,6 +127,17 @@ public class IridaApiServicesConfig { private int analysisTaskThreads; @Value("${locales.enabled}") private String availableLocales; + + @Value("${irida.storage.type}") + private String storageType; + + @Value("${azure.container.name}") + private String containerName; + + @Value("${azure.account.connection.string}") + private String connectionStr; + + @Autowired private IridaPluginConfig.IridaPluginList pipelinePlugins; @@ -282,6 +295,12 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { return null; } + @Bean(name = "iridaFileStorageService") + public IridaFileStorageServiceImpl iridaFileStorageService() { + IridaFileStorageServiceImpl iridaFileStorageService = new IridaFileStorageServiceImpl(storageType, connectionStr, containerName); + return iridaFileStorageService; + } + @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, QCEntryRepository qcRepository, GzipFileProcessor gzipFileProcessor, diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index 911b7724445..05a1786b427 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -26,7 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.MutableIridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; /** * A reference file to be associated with a {@link Project}. @@ -39,8 +39,6 @@ @EntityListeners({AuditingEntityListener.class, RelativePathTranslatorListener.class}) public class ReferenceFile implements VersionedFileFields, MutableIridaThing { - private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -94,7 +92,7 @@ public ReferenceFile(Path file) { @Override public String getLabel() { - return fileService.getFileName(file); + return file.getFileName().toString(); } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java new file mode 100644 index 00000000000..73d9ce70265 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java @@ -0,0 +1,48 @@ +package ca.corefacility.bioinformatics.irida.model.sequenceFile; + +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import ca.corefacility.bioinformatics.irida.model.IridaThing; +import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class CloudSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { + private static final Logger logger = LoggerFactory.getLogger(CloudSequenceFile.class); + + @Autowired + public IridaFileStorageServiceImpl iridaFileStorageService; + + private Path file; + public CloudSequenceFile() { + super(); + } + + public CloudSequenceFile(Path sampleFile) { + super(sampleFile); + this.file = sampleFile; + } + + @Override + public String getLabel() { + return "testnewfile.txt"; + } + /** + * Get the size of the file. + * + * @return The String representation of the file size + */ + @JsonIgnore + @Override + public String getFileSize() { + String size = "N/A"; + size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageService.getFileSize(file), true); + return size; + } + +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java new file mode 100644 index 00000000000..5eee1425b7f --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java @@ -0,0 +1,51 @@ +package ca.corefacility.bioinformatics.irida.model.sequenceFile; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ca.corefacility.bioinformatics.irida.model.IridaThing; +import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class LocalSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { + private static final Logger logger = LoggerFactory.getLogger(LocalSequenceFile.class); + + private Path file; + public LocalSequenceFile() { + super(); + } + + public LocalSequenceFile(Path sampleFile) { + super(sampleFile); + this.file = sampleFile; + } + + @Override + public String getLabel() { + return file.getFileName().toString(); + } + /** + * Get the size of the file. + * + * @return The String representation of the file size + */ + @JsonIgnore + @Override + public String getFileSize() { + String size = "N/A"; + try { + size = IridaSequenceFile.humanReadableByteCount(Files.size(file), true); + } catch (NoSuchFileException e) { + logger.error("Could not find file " + file); + } catch (IOException e) { + logger.error("Could not calculate file size: ", e); + } + return size; + } + +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index abc0c2e1fd2..73b90e538ac 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -29,7 +29,6 @@ import org.hibernate.envers.NotAudited; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; @@ -44,7 +43,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -58,7 +57,7 @@ @Table(name = "sequence_file") @Audited @EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class }) -public class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, +public abstract class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, VersionedFileFields, IridaSequenceFile, RemoteSynchronizable { private static final Logger logger = LoggerFactory.getLogger(SequenceFile.class); @@ -106,8 +105,6 @@ public class SequenceFile extends IridaResourceSupport implements MutableIridaTh @JoinColumn(name = "remote_status") private RemoteStatus remoteStatus; - private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); - public SequenceFile() { createdDate = new Date(); fileRevisionNumber = 0L; @@ -157,11 +154,6 @@ public void setFile(Path file) { this.file = file; } - @Override - public String getLabel() { - return fileService.getFileName(file); - } - @Override public Long getId() { return id; @@ -220,17 +212,6 @@ public String getOptionalProperty(String key) { return optionalProperties.get(key); } - /** - * Get the size of the file. - * - * @return The String representation of the file size - */ - @JsonIgnore - public String getFileSize() { - String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(fileService.getFileSize(file), true); - return size; - } /** * Set the Map of optional properties diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 7556305c395..c248da8f2aa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -26,7 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; /** * Store file references to files produced by a workflow execution that we @@ -65,8 +65,6 @@ public class AnalysisOutputFile extends IridaResourceSupport implements IridaThi @Column(name = "label_prefix") private final String labelPrefix; - private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); - /** * for hibernate */ @@ -133,7 +131,7 @@ public void incrementFileRevisionNumber() { @Override public String getLabel() { - return Strings.isNullOrEmpty(labelPrefix) ? fileService.getFileName(file) : labelPrefix + '-' + fileService.getFileName(file); + return Strings.isNullOrEmpty(labelPrefix) ? file.toFile().getName() : labelPrefix + '-' + file.toFile().getName(); } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 418ab50d0d8..1a75817c3d6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -5,17 +5,23 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; + /** * {@link SequencingObjectConcatenator} for {@link SequenceFilePair}s */ public class SequenceFilePairConcatenator extends SequencingObjectConcatenator { - + + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + public SequenceFilePairConcatenator() { } @@ -61,8 +67,8 @@ public SequenceFilePair concatenateFiles(List toConc } // create new SequenceFiles - SequenceFile forward = new SequenceFile(forwardFile); - SequenceFile reverse = new SequenceFile(reverseFile); + SequenceFile forward = iridaFileStorageFactory.createSequenceFile(forwardFile); + SequenceFile reverse = iridaFileStorageFactory.createSequenceFile(reverseFile); // create the new pair SequenceFilePair sequenceFilePair = new SequenceFilePair(forward, reverse); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index f94732a630f..b04d65f668b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,17 +5,23 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; + /** * {@link SequenceFilePairConcatenator} for {@link SingleEndSequenceFile}s */ public class SingleEndSequenceFileConcatenator extends SequencingObjectConcatenator { + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + public SingleEndSequenceFileConcatenator() { } @@ -53,7 +59,7 @@ public SingleEndSequenceFile concatenateFiles(List t } // create the new sequencefile and object - SequenceFile forward = new SequenceFile(tempFile); + SequenceFile forward = iridaFileStorageFactory.createSequenceFile(tempFile); SingleEndSequenceFile seqObject = new SingleEndSequenceFile(forward); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index af25dc23382..58aa5978e1a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -10,7 +10,8 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.service.IridaFileStorageService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,7 @@ public class FastqcFileProcessor implements FileProcessor { private final SequenceFileRepository sequenceFileRepository; private final AnalysisOutputFileRepository outputFileRepository; private final MessageSource messageSource; + private IridaFileStorageServiceImpl iridaFileStorageService; private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); @@ -66,10 +68,11 @@ public class FastqcFileProcessor implements FileProcessor { */ @Autowired public FastqcFileProcessor(final MessageSource messageSource, final SequenceFileRepository sequenceFileRepository, - AnalysisOutputFileRepository outputFileRepository) { + AnalysisOutputFileRepository outputFileRepository, IridaFileStorageServiceImpl iridaFileStorageService) { this.messageSource = messageSource; this.sequenceFileRepository = sequenceFileRepository; this.outputFileRepository = outputFileRepository; + this.iridaFileStorageService = iridaFileStorageService; } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index c2b1702cb97..06f1f698c3a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -21,14 +21,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.ReflectionUtils; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageServiceImpl; /** * Custom implementation of a repository that writes the {@link Path} part of an diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java similarity index 92% rename from src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java rename to src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java index 870c0a96213..98bbee7aa55 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java @@ -1,4 +1,4 @@ -package ca.corefacility.bioinformatics.irida.service.impl; +package ca.corefacility.bioinformatics.irida.repositories.filesystem; import java.io.File; import java.io.IOException; @@ -26,22 +26,24 @@ public class IridaFileStorageServiceImpl implements IridaFileStorageService { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageServiceImpl.class); - @Value("${irida.storage.type}") private String storageType; - @Value("${azure.container.name}") + //Azure Specific Variables private String containerName; - - @Value("${azure.account.connection.string}") - private String connectStr; - + private String connectionStr; private BlobServiceClient blobServiceClient; - private BlobContainerClient containerClient; + private BlobContainerClient containerClient ; private BlobClient blobClient; + //AWS Specific Variables + @Autowired - public IridaFileStorageServiceImpl(){ - this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectStr) + public IridaFileStorageServiceImpl(String storageType, String connectionStr, String containerName){ + this.storageType = storageType; + this.containerName = containerName; + this.connectionStr = connectionStr; + + this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); } @@ -175,9 +177,6 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - - logger.debug("STORAGE TYPE IS: " + storageType); - if(storageType.equalsIgnoreCase("azure")) { blobClient = containerClient.getBlobClient(file.toAbsolutePath() .toString() diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index e81ca6c3360..2d4c44e7af2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -54,6 +54,7 @@ import ca.corefacility.bioinformatics.irida.ria.web.BaseController; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleDetails; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; @@ -111,6 +112,9 @@ public class SamplesController extends BaseController { private final MessageSource messageSource; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Autowired public SamplesController(SampleService sampleService, ProjectService projectService, SequencingObjectService sequencingObjectService, UpdateSamplePermission updateSamplePermission, @@ -658,7 +662,7 @@ private SequenceFile createSequenceFile(MultipartFile file) throws IOException { Path temp = Files.createTempDirectory(null); Path target = temp.resolve(file.getOriginalFilename()); file.transferTo(target.toFile()); - return new SequenceFile(target); + return iridaFileStorageFactory.createSequenceFile(target); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java new file mode 100644 index 00000000000..826a37d8502 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java @@ -0,0 +1,43 @@ +package ca.corefacility.bioinformatics.irida.service.impl; + +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; + +@Service +public class IridaFileStorageFactoryImpl { + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageFactoryImpl.class); + + @Autowired + private IridaFileStorageServiceImpl iridaFileStorageService; + + @Autowired + public IridaFileStorageFactoryImpl(){ + } + + public SequenceFile createSequenceFile(Path file) { + if(iridaFileStorageService.storageTypeIsLocal()) { + return new LocalSequenceFile(file); + } else { + return new CloudSequenceFile(file); + } + } + + public SequenceFile createEmptySequenceFile(){ + if(iridaFileStorageService.storageTypeIsLocal()) { + return new LocalSequenceFile(); + } else { + return new CloudSequenceFile(); + } + } + +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index cbab1eb258d..edc3c4d136e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -37,7 +37,7 @@ @Service public class SequencingObjectServiceImpl extends CRUDServiceImpl implements SequencingObjectService { - + private final SampleSequencingObjectJoinRepository ssoRepository; private final SequenceFileRepository sequenceFileRepository; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java index b07bed26f6e..f9e79c62ec1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java @@ -1,6 +1,9 @@ package ca.corefacility.bioinformatics.irida.web.assembler.resource.sequencefile; +import org.springframework.beans.factory.annotation.Autowired; + import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -18,8 +21,11 @@ public class SequenceFileResource { private SequenceFile resource; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + public SequenceFileResource() { - resource = new SequenceFile(); + resource = iridaFileStorageFactory.createEmptySequenceFile(); } public SequenceFileResource(SequenceFile sequenceFile) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 9b18a7db64c..75999fd11dc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -37,6 +37,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.service.AnalysisService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -135,6 +136,9 @@ public class RESTSampleSequenceFilesController { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + protected RESTSampleSequenceFilesController() { } @@ -414,7 +418,7 @@ public ModelMap addNewSequenceFileToSample(@PathVariable Long sampleId, @Request logger.trace("Read miseq run " + miseqRunId); } } else { - sf = new SequenceFile(); + sf = iridaFileStorageFactory.createEmptySequenceFile(); } sf.setFile(target); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java index 4b7d43673d6..de1e20af6c9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.User; @@ -121,7 +122,7 @@ public void testHandleSequenceFileAddedEventSingle() { Class clazz = DataAddedToSampleProjectEvent.class; Project project = new Project(); Sample sample = new Sample(); - SequenceFile file = new SequenceFile(); + SequenceFile file = new LocalSequenceFile(); SingleEndSequenceFile seqObj = new SingleEndSequenceFile(file); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(sample, seqObj); @@ -151,7 +152,7 @@ public void testHandleSequenceFileAddedEventMultipleReturn() { Class clazz = DataAddedToSampleProjectEvent.class; Project project = new Project(); Sample sample = new Sample(); - SequenceFile file = new SequenceFile(); + SequenceFile file = new LocalSequenceFile(); SingleEndSequenceFile seqObj1 = new SingleEndSequenceFile(file); SingleEndSequenceFile seqObj2 = new SingleEndSequenceFile(file); SampleSequencingObjectJoin join1 = new SampleSequencingObjectJoin(sample, seqObj1); @@ -184,7 +185,7 @@ public void testHandleSequenceFileAddedEventMultipleProjects() { Project project = new Project("p1"); Project project2 = new Project("p2"); Sample sample = new Sample(); - SequenceFile file = new SequenceFile(); + SequenceFile file = new LocalSequenceFile(); SingleEndSequenceFile seqObj = new SingleEndSequenceFile(file); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(sample, seqObj); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java index 1672858ca34..8a1da220717 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java @@ -10,6 +10,7 @@ import org.junit.Before; import org.junit.Test; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; @@ -49,11 +50,11 @@ public void setup() throws IOException { reversePathGood = tempDir.resolve("Test_R2_001.fastq"); reversePathBad = tempDir.resolve("Test_B.fastq"); - sequenceFileForwardGood = new SequenceFile(forwardPathGood); - sequenceFileForwardBad = new SequenceFile(forwardPathBad); + sequenceFileForwardGood = new LocalSequenceFile(forwardPathGood); + sequenceFileForwardBad = new LocalSequenceFile(forwardPathBad); - sequenceFileReverseGood = new SequenceFile(reversePathGood); - sequenceFileReverseBad = new SequenceFile(reversePathBad); + sequenceFileReverseGood = new LocalSequenceFile(reversePathGood); + sequenceFileReverseBad = new LocalSequenceFile(reversePathBad); sequenceFilePairGood = new SequenceFilePair(sequenceFileForwardGood, sequenceFileReverseGood); sequenceFilePairBad = new SequenceFilePair(sequenceFileForwardBad, sequenceFileReverseBad); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java index b13f849c539..4290f702f38 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java @@ -11,6 +11,7 @@ import com.google.common.collect.Sets; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -23,7 +24,7 @@ */ public class AnalysisSubmissionTest { - private static final SequenceFile sequenceFile = new SequenceFile(); + private static final SequenceFile sequenceFile = new LocalSequenceFile(); private static final SingleEndSequenceFile singleEndFile = new SingleEndSequenceFile(sequenceFile); private static final ReferenceFile referenceFile = new ReferenceFile(); private static final Map inputParameters = ImmutableMap.of("test", "test"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 06254aac387..9086c28d77a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -1,8 +1,10 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; + import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; @@ -19,7 +21,7 @@ public class SequenceFilePairConcatenatorTest { private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" + SEQUENCE + "\n+\n?????????").getBytes(); - SequenceFilePairConcatenator concat; + private SequenceFilePairConcatenator concat; @Before public void setUp() { @@ -63,6 +65,6 @@ private SequenceFile createSequenceFile(String name) throws IOException { Path sequenceFile = Files.createTempFile(name, ".fastq"); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - return new SequenceFile(sequenceFile); + return new LocalSequenceFile(sequenceFile); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index d40f09f63cd..794f4323d36 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import com.google.common.collect.Lists; @@ -110,6 +111,6 @@ private SequenceFile createSequenceFile(String name, String extension) throws IO Path sequenceFile = Files.createTempFile(name, extension); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - return new SequenceFile(sequenceFile); + return new LocalSequenceFile(sequenceFile); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java index 234a413517b..779a3877604 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.user.User; @@ -111,8 +111,8 @@ public void setUp() throws IridaWorkflowNotFoundException { @Test public void testAssembleFile() { Long sequenceFileId = 1L; - SequenceFilePair pair = new SequenceFilePair(new SequenceFile(Paths.get("file_R1_1.fastq.gz")), - new SequenceFile(Paths.get("file_R2_1.fastq.gz"))); + SequenceFilePair pair = new SequenceFilePair(new LocalSequenceFile(Paths.get("file_R1_1.fastq.gz")), + new LocalSequenceFile(Paths.get("file_R2_1.fastq.gz"))); Sample sample = new Sample(); Project project = new Project(); @@ -152,8 +152,8 @@ public void testAssembleFile() { @Test public void testOtherWorkflow() { Long sequenceFileId = 1L; - SequenceFilePair pair = new SequenceFilePair(new SequenceFile(Paths.get("file_R1_1.fastq.gz")), - new SequenceFile(Paths.get("file_R2_1.fastq.gz"))); + SequenceFilePair pair = new SequenceFilePair(new LocalSequenceFile(Paths.get("file_R1_1.fastq.gz")), + new LocalSequenceFile(Paths.get("file_R2_1.fastq.gz"))); Sample sample = new Sample(); Project project = new Project(); @@ -203,8 +203,8 @@ public void testNoSubmissions() { @Test public void testOneProjectEnabled() { - SequenceFilePair pair = new SequenceFilePair(new SequenceFile(Paths.get("file_R1_1.fastq.gz")), - new SequenceFile(Paths.get("file_R2_1.fastq.gz"))); + SequenceFilePair pair = new SequenceFilePair(new LocalSequenceFile(Paths.get("file_R1_1.fastq.gz")), + new LocalSequenceFile(Paths.get("file_R2_1.fastq.gz"))); Sample sample = new Sample(); Project project = new Project("assemble me"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index ab25ff88a07..3bfdef81c91 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -13,6 +13,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -49,7 +50,7 @@ public void testChecksumCreated() throws IOException { @Test(expected = FileProcessorException.class) public void testFileNotExists() throws IOException { - final SequenceFile sf = new SequenceFile(Paths.get("/reallyfakefile")); + final SequenceFile sf = new LocalSequenceFile(Paths.get("/reallyfakefile")); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -57,7 +58,7 @@ public void testFileNotExists() throws IOException { } private SequenceFile constructSequenceFile() throws IOException { - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); Path sequenceFile = Files.createTempFile(null, null); Files.write(sequenceFile, FILE_CONTENTS.getBytes()); sf.setFile(sequenceFile); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java index 8fb66583964..ad6cbf1488b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java @@ -17,6 +17,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.CoverageQCEntry; import ca.corefacility.bioinformatics.irida.model.sample.QCEntry; import ca.corefacility.bioinformatics.irida.model.sample.QCEntry.QCEntryStatus; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -44,7 +45,7 @@ public void testGoodCoverage() { Project p = new Project(); p.setGenomeSize(100L); p.setMinimumCoverage(2); - SequenceFile file = new SequenceFile(); + SequenceFile file = new LocalSequenceFile(); SequencingObject o = new SingleEndSequenceFile(file); AnalysisFastQC fqc = mock(AnalysisFastQC.class); Long baseCount = 300L; @@ -71,7 +72,7 @@ public void testBadCoverage() { Project p = new Project(); p.setGenomeSize(100L); p.setMinimumCoverage(5); - SequenceFile file = new SequenceFile(); + SequenceFile file = new LocalSequenceFile(); SequencingObject o = new SingleEndSequenceFile(file); AnalysisFastQC fqc = mock(AnalysisFastQC.class); Long baseCount = 300L; @@ -98,7 +99,7 @@ public void testRemoveExistingEntry() { Project p = new Project(); p.setGenomeSize(100L); p.setMinimumCoverage(2); - SequenceFile file = new SequenceFile(); + SequenceFile file = new LocalSequenceFile(); SequencingObject o = new SingleEndSequenceFile(file); AnalysisFastQC fqc = mock(AnalysisFastQC.class); Long baseCount = 300L; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 0db5ebd1086..d9f6db10fcd 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -16,6 +16,7 @@ import java.nio.file.Path; import java.util.Iterator; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import org.junit.Before; @@ -32,6 +33,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -43,6 +45,7 @@ public class FastqcFileProcessorTest { private FastqcFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private AnalysisOutputFileRepository outputFileRepository; + private IridaFileStorageServiceImpl iridaFileStorageService; private MessageSource messageSource; private static final Logger logger = LoggerFactory.getLogger(FastqcFileProcessorTest.class); @@ -56,7 +59,7 @@ public void setUp() { messageSource = mock(MessageSource.class); sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); - fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository); + fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageService); } @Test(expected = FileProcessorException.class) @@ -65,7 +68,7 @@ public void testHandleFastaFile() throws IOException { // dummy), but that's A-OK. Path fasta = Files.createTempFile(null, null); Files.write(fasta, FASTA_FILE_CONTENTS.getBytes()); - SequenceFile sf = new SequenceFile(fasta); + SequenceFile sf = new LocalSequenceFile(fasta); sf.setId(1L); Runtime.getRuntime().addShutdownHook(new DeleteFileOnExit(fasta)); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -82,7 +85,7 @@ public void testHandleFastqFile() throws IOException, IllegalArgumentException, ArgumentCaptor argument = ArgumentCaptor.forClass(SequenceFile.class); - SequenceFile sf = new SequenceFile(fastq); + SequenceFile sf = new LocalSequenceFile(fastq); sf.setId(1L); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); try { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 8edbeea806c..3b1fef73a21 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -18,6 +18,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -35,6 +36,7 @@ public class GzipFileProcessorTest { private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; + @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); @@ -101,7 +103,7 @@ public void handleCompressedFileWithGzExtension() throws IOException { // the file processor should decompress the file, then update the // sequence file in the database. SequenceFile sf = constructSequenceFile(); - SequenceFile sfUpdated = new SequenceFile(); + SequenceFile sfUpdated = new LocalSequenceFile(); sfUpdated.setFile(sf.getFile()); final Long id = 1L; sf.setId(id); @@ -138,7 +140,7 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { // the file processor should decompress the file, then update the // sequence file in the database. SequenceFile sf = constructSequenceFile(); - SequenceFile sfUpdated = new SequenceFile(); + SequenceFile sfUpdated = new LocalSequenceFile(); sfUpdated.setFile(sf.getFile()); final Long id = 1L; sf.setId(id); @@ -170,7 +172,7 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { } private SequenceFile constructSequenceFile() throws IOException { - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); Path sequenceFile = Files.createTempFile(null, null); Files.write(sequenceFile, FILE_CONTENTS.getBytes()); sf.setFile(sequenceFile); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index 8f5350f9315..0d37677d01b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -19,6 +19,7 @@ import org.junit.Before; import org.junit.Test; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepositoryImpl; import ca.corefacility.bioinformatics.irida.util.RecursiveDeleteVisitor; @@ -52,7 +53,7 @@ private Path getTempFile() throws IOException { @Test public void testCreateFileMissingIdentifier() throws IOException { - SequenceFile s = new SequenceFile(getTempFile()); + SequenceFile s = new LocalSequenceFile(getTempFile()); try { repository.save(s); fail(); @@ -67,7 +68,7 @@ public void testCreateFile() throws IOException { Long lid = new Long(1111); Path f = getTempFile(); String filename = f.getFileName().toString(); - SequenceFile s = new SequenceFile(f); + SequenceFile s = new LocalSequenceFile(f); s.setId(lid); when(entityManager.find(SequenceFile.class, lid)).thenReturn(s); when(entityManager.merge(s)).thenReturn(s); @@ -84,7 +85,7 @@ public void testCreateFile() throws IOException { @Test public void testUpdateFileMissingIdentifier() throws IOException { - SequenceFile s = new SequenceFile(getTempFile()); + SequenceFile s = new LocalSequenceFile(getTempFile()); try { repository.save(s); fail(); @@ -97,7 +98,7 @@ public void testUpdateFileMissingIdentifier() throws IOException { @Test public void testUpdateMissingDirectory() throws IOException { Path f = getTempFile(); - SequenceFile s = new SequenceFile(f); + SequenceFile s = new LocalSequenceFile(f); try { repository.save(s); @@ -115,7 +116,7 @@ public void testUpdateExistingFilename() throws IOException { Long lid = new Long(1111); Path oldFile = getTempFile(); Files.write(oldFile, originalText.getBytes()); - SequenceFile sf = new SequenceFile(oldFile); + SequenceFile sf = new LocalSequenceFile(oldFile); sf.setId(lid); // create the directory and put the file into it. // so call create instead of rewriting the logic: @@ -158,7 +159,7 @@ public void testUpdateExistingFilename() throws IOException { public void testUpdate() throws IOException { Long lId = new Long(9999); Path originalFile = getTempFile(); - SequenceFile original = new SequenceFile(originalFile); + SequenceFile original = new LocalSequenceFile(originalFile); original.setId(lId); when(entityManager.find(SequenceFile.class, lId)).thenReturn(original); when(entityManager.merge(original)).thenReturn(original); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java index d83281ba507..f216079999b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java @@ -33,6 +33,7 @@ import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -79,7 +80,7 @@ public static Sample constructSample() { public static SingleEndSequenceFile constructSingleEndSequenceFile(){ Path path = Paths.get("/tmp/sequence-files/fake-file1.fast"); - return new SingleEndSequenceFile(new SequenceFile(path)); + return new SingleEndSequenceFile(new LocalSequenceFile(path)); } /** @@ -145,7 +146,7 @@ public static List generateSequencingObjectsForSampl List join = new ArrayList<>(); for (long i = 0; i < 5; i++) { Path path = Paths.get("/tmp/sequence-files/fake-file" + Math.random() + ".fast"); - SequenceFile file = new SequenceFile(path); + SequenceFile file = new LocalSequenceFile(path); file.setId(i); SingleEndSequenceFile obj = new SingleEndSequenceFile(file); obj.setId(i); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java index dff3d75c572..b0198f03f89 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java @@ -1,7 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.unit.web; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.SequencingRunController; @@ -24,6 +24,7 @@ public class SequencingRunControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; + @Before public void setup() { sequencingRunService = mock(SequencingRunService.class); @@ -57,7 +58,7 @@ public void testGetFilesPage() throws IOException { ExtendedModelMap model = new ExtendedModelMap(); SequencingRun sequencingRunEntity = new SequencingRun(SequencingRun.LayoutType.PAIRED_END, "miseq"); - ImmutableSet files = ImmutableSet.of(new SingleEndSequenceFile(new SequenceFile())); + ImmutableSet files = ImmutableSet.of(new SingleEndSequenceFile(new LocalSequenceFile())); when(sequencingRunService.read(runId)).thenReturn(sequencingRunEntity); when(objectService.getSequencingObjectsForSequencingRun(sequencingRunEntity)).thenReturn(files); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index 3b844b36398..d5ece606df7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -20,6 +20,7 @@ import org.springframework.ui.ExtendedModelMap; import org.springframework.ui.Model; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; @@ -51,7 +52,7 @@ public void setUp() { controller = new SequenceFileController(objectService, sequencingRunService, analysisService); Path path = Paths.get(FILE_PATH); - SequenceFile file = new SequenceFile(path); + SequenceFile file = new LocalSequenceFile(path); file.setId(FILE_ID); SingleEndSequenceFile seqObject = new SingleEndSequenceFile(file); when(objectService.read(anyLong())).thenReturn(seqObject); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java index 827ade49845..390b722131a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java @@ -32,6 +32,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.Role; @@ -47,7 +48,6 @@ import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; -import ca.corefacility.bioinformatics.irida.service.user.UserService; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -328,7 +328,7 @@ public void testDownloadSamples() throws IOException { MockHttpServletResponse response = new MockHttpServletResponse(); Path path = Paths.get(FILE_PATH); - SequenceFile file = new SequenceFile(path); + SequenceFile file = new LocalSequenceFile(path); ImmutableList filejoin = ImmutableList.of(new SampleSequencingObjectJoin(sample, new SingleEndSequenceFile(file))); @@ -362,7 +362,7 @@ public void testDownloadSamplesWithSameName() throws IOException { MockHttpServletResponse response = new MockHttpServletResponse(); Path path = Paths.get(FILE_PATH); - SequenceFile file = new SequenceFile(path); + SequenceFile file = new LocalSequenceFile(path); ImmutableList filejoin = ImmutableList.of(new SampleSequencingObjectJoin(sample, new SingleEndSequenceFile(file)), new SampleSequencingObjectJoin(sample, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index a21e40ce4cd..8c8e6362e14 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -46,10 +46,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -150,7 +147,7 @@ public void testGetSampleFiles() throws IOException { ExtendedModelMap model = new ExtendedModelMap(); Long sampleId = 1L; Sample sample = new Sample(); - SequenceFile file = new SequenceFile(Paths.get("/tmp")); + SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); file.setId(2L); Project project = new Project(); @@ -179,7 +176,7 @@ public void testGetSampleFilesAsAdmin() throws IOException { ExtendedModelMap model = new ExtendedModelMap(); Long sampleId = 1L; Sample sample = new Sample(); - SequenceFile file = new SequenceFile(Paths.get("/tmp")); + SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); file.setId(2L); List files = Lists.newArrayList(new SampleSequencingObjectJoin(sample, @@ -209,7 +206,7 @@ public void testGetSampleFilesNoAccess() throws IOException { Long sampleId = 1L; Sample sample = new Sample(); - SequenceFile file = new SequenceFile(Paths.get("/tmp")); + SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); file.setId(2L); Project project = new Project(); @@ -240,7 +237,7 @@ public void testRemoveFileFromSample() { Long sampleId = 1L; Long fileId = 2L; Sample sample = new Sample(); - SequencingObject file = new SingleEndSequenceFile(new SequenceFile(Paths.get("/tmp"))); + SequencingObject file = new SingleEndSequenceFile(new LocalSequenceFile(Paths.get("/tmp"))); when(sampleService.read(sampleId)).thenReturn(sample); when(sequencingObjectService.readSequencingObjectForSample(sample, fileId)).thenReturn(file); @@ -258,7 +255,7 @@ public void testDownloadAssembly() throws IOException { Long sampleId = 1L; Long assemblyId = 3L; - SequenceFile file = new SequenceFile(Paths.get("/tmp")); + SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); file.setId(2L); Sample sample = new Sample(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java index c42452189e5..80b9a97ec83 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java @@ -20,10 +20,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -85,7 +82,7 @@ public void setUp() { readAnalysisSubmissionPermission = new ReadAnalysisSubmissionPermission(analysisSubmissionRepository, userRepository, sequencingObjectRepository, seqObjectPermission, pasRepository, readProjectPermission); - inputSingleFiles = Sets.newHashSet(new SingleEndSequenceFile(new SequenceFile())); + inputSingleFiles = Sets.newHashSet(new SingleEndSequenceFile(new LocalSequenceFile())); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java index 3d372078ae2..8a9873b4986 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java @@ -19,15 +19,14 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import org.springframework.beans.factory.annotation.Autowired; + import com.google.common.collect.Sets; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowState; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -466,7 +465,7 @@ public List setupSequencingObjectInDatabase(Long sampleId List returnedSequenceFiles = new ArrayList<>(); for (Path sequenceFilePath : sequenceFilePaths) { - SingleEndSequenceFile singleEndSequenceFile = new SingleEndSequenceFile(new SequenceFile(sequenceFilePath)); + SingleEndSequenceFile singleEndSequenceFile = new SingleEndSequenceFile(new LocalSequenceFile(sequenceFilePath)); sequencingObjectService.createSequencingObjectInSample(singleEndSequenceFile, sample); waitForFilesToSettle(singleEndSequenceFile); @@ -496,8 +495,8 @@ public List setupSampleSequenceFileInDatabase(long sampleId, L Sample sample = sampleService.read(sampleId); List returnedSequenceFilePairs = new ArrayList<>(); for (int i = 0; i < sequenceFiles1.size(); i++) { - SequenceFile sf1 = new SequenceFile(sequenceFiles1.get(i)); - SequenceFile sf2 = new SequenceFile(sequenceFiles2.get(i)); + SequenceFile sf1 = new LocalSequenceFile(sequenceFiles1.get(i)); + SequenceFile sf2 = new LocalSequenceFile(sequenceFiles2.get(i)); SequenceFilePair pair = new SequenceFilePair(sf1, sf2); sequencingObjectService.createSequencingObjectInSample(pair, sample); waitForFilesToSettle(pair); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java index 2bca7cb7d5a..1ed4edc6c9c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java @@ -33,7 +33,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.galaxy.WorkflowUploadException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; @@ -130,7 +130,7 @@ public void setup() throws IridaWorkflowNotFoundException, IOException, Executio MockitoAnnotations.initMocks(this); String submissionName = "name"; - Set submissionInputFiles = Sets.newHashSet(new SingleEndSequenceFile(new SequenceFile())); + Set submissionInputFiles = Sets.newHashSet(new SingleEndSequenceFile(new LocalSequenceFile())); analysisSubmission = AnalysisSubmission.builder(WORKFLOW_ID).name(submissionName + "intial").inputFiles(submissionInputFiles).build(); analysisPreparing = AnalysisSubmission.builder(WORKFLOW_ID).name(submissionName + "preparing").inputFiles(submissionInputFiles).build(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java index 5ccd78dac03..7e8bbf314e6 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java @@ -66,10 +66,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.WorkflowException; import ca.corefacility.bioinformatics.irida.exceptions.galaxy.GalaxyDatasetNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; @@ -221,7 +218,7 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc Files.delete(referenceFilePath); Files.copy(referenceFilePathReal, referenceFilePath); - singleFileSet = Sets.newHashSet(new SingleEndSequenceFile(new SequenceFile(sequenceFilePathA))); + singleFileSet = Sets.newHashSet(new SingleEndSequenceFile(new LocalSequenceFile(sequenceFilePathA))); GalaxyInstance galaxyInstanceAdmin = localGalaxy.getGalaxyInstanceAdmin(); HistoriesClient historiesClient = galaxyInstanceAdmin.getHistoriesClient(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index fb63f03ea5c..3e0a8187312 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -5,10 +5,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.galaxy.GalaxyDatasetException; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.upload.galaxy.GalaxyProjectName; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflowTestBuilder; @@ -154,9 +151,9 @@ public class AnalysisWorkspaceServiceGalaxyTest { public void setup() throws IOException, UploadException, GalaxyDatasetException { MockitoAnnotations.initMocks(this); - sFileA = new SequenceFile(createTempFile("fileA", "fastq")); - sFileB = new SequenceFile(createTempFile("fileB", "fastq")); - sFileC = new SequenceFile(createTempFile("fileC", "fastq")); + sFileA = new LocalSequenceFile(createTempFile("fileA", "fastq")); + sFileB = new LocalSequenceFile(createTempFile("fileB", "fastq")); + sFileC = new LocalSequenceFile(createTempFile("fileC", "fastq")); sObjA = new SingleEndSequenceFile(sFileA); sObjB = new SingleEndSequenceFile(sFileB); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java index 3f032eb2b54..4802315b328 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java @@ -8,6 +8,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.service.impl.TestEmailController; @@ -364,7 +365,7 @@ private NcbiExportSubmission createFakeSubmission(String sequenceFileExtension) NcbiBioSampleFiles ncbiBioSampleFiles = new NcbiBioSampleFiles(); Path tempFile = Files.createTempFile("sequencefile", sequenceFileExtension); - SequenceFile sequenceFile = new SequenceFile(tempFile); + SequenceFile sequenceFile = new LocalSequenceFile(tempFile); sequenceFile.setId(1L); SingleEndSequenceFile singleFile = new SingleEndSequenceFile(sequenceFile); singleFile.setId(1L); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java index ab975040874..999920cad1e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java @@ -2,6 +2,7 @@ import ca.corefacility.bioinformatics.irida.model.NcbiExportSubmission; import ca.corefacility.bioinformatics.irida.model.export.NcbiBioSampleFiles; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -37,7 +38,7 @@ public void setup() { @Test public void testCreate() { - SingleEndSequenceFile sequenceFile = new SingleEndSequenceFile(new SequenceFile()); + SingleEndSequenceFile sequenceFile = new SingleEndSequenceFile(new LocalSequenceFile()); NcbiBioSampleFiles ncbiBioSampleFiles = new NcbiBioSampleFiles("sample", Sets.newHashSet(sequenceFile), Sets.newHashSet(), null, "library_name", null, null, null, "library_construction_protocol", @@ -52,7 +53,7 @@ public void testCreate() { @Test public void testCreatePairs() { - SequenceFile sequenceFile = new SequenceFile(); + SequenceFile sequenceFile = new LocalSequenceFile(); NcbiBioSampleFiles ncbiBioSampleFiles = new NcbiBioSampleFiles("sample", Sets.newHashSet(), Sets.newHashSet(new SequenceFilePair(sequenceFile, sequenceFile)), null, "library_name", null, null, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java index 93f0563d1be..c80eb498b33 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java @@ -60,19 +60,12 @@ import ca.corefacility.bioinformatics.irida.model.sample.QCEntry.QCEntryStatus; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; -import ca.corefacility.bioinformatics.irida.service.AnalysisService; -import ca.corefacility.bioinformatics.irida.service.ProjectService; -import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; -import ca.corefacility.bioinformatics.irida.service.SequencingRunService; +import ca.corefacility.bioinformatics.irida.service.*; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @RunWith(SpringJUnit4ClassRunner.class) @@ -175,7 +168,7 @@ public void testGetSequenceFilePairForSample() { @WithMockUser(username = "fbristow", roles = "SEQUENCER") public void testCreateSequenceFileInSample() throws IOException, InterruptedException { Sample s = sampleService.read(1L); - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = new GZIPOutputStream(Files.newOutputStream(sequenceFile)); gzOut.write(FASTQ_FILE_CONTENTS); @@ -207,7 +200,7 @@ public void testCreateSequenceFileInSample() throws IOException, InterruptedExce @WithMockUser(username = "fbristow", roles = "SEQUENCER") public void testCreateCorruptSequenceFileInSample() throws IOException, InterruptedException { Sample s = sampleService.read(1L); - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = Files.newOutputStream(sequenceFile); gzOut.write("not a file".getBytes()); @@ -331,7 +324,7 @@ public void testCreateNotCompressedSequenceFile() throws IOException, Interrupte @WithMockUser(username = "fbristow", roles = "SEQUENCER") public void testCreateCompressedSequenceFile() throws IOException, InterruptedException { final Long expectedRevisionNumber = 2L; - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = new GZIPOutputStream(Files.newOutputStream(sequenceFile)); gzOut.write(FASTQ_FILE_CONTENTS); @@ -429,7 +422,7 @@ public void testCoverage() throws IOException, InterruptedException { project = projectService.update(project); Sample s = sampleService.read(1L); - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = new GZIPOutputStream(Files.newOutputStream(sequenceFile)); gzOut.write(FASTQ_FILE_CONTENTS); @@ -572,6 +565,6 @@ private SequenceFile createSequenceFile(String name) throws IOException{ Path sequenceFile = Files.createTempFile(name, ".fastq"); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - return new SequenceFile(sequenceFile); + return new LocalSequenceFile(sequenceFile); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java index d9074ac39a2..b72f5a24b92 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java @@ -6,6 +6,7 @@ import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun.LayoutType; import ca.corefacility.bioinformatics.irida.model.sample.Sample; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -76,6 +77,7 @@ public class SequencingRunServiceImplIT { @Autowired private AnalysisService analysisService; + @Test @WithMockUser(username = "fbristow", password = "password1", roles = "SEQUENCER") public void testAddSequenceFileToMiseqRunAsSequencer() throws IOException, InterruptedException { @@ -110,7 +112,7 @@ private void testAddSequenceFileToMiseqRun() throws IOException, InterruptedExce // that we can link to. Path sequenceFile = Files.createTempFile(null, null); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - SequenceFile sf = new SequenceFile(sequenceFile); + SequenceFile sf = new LocalSequenceFile(sequenceFile); SequencingObject so = new SingleEndSequenceFile(sf); so = objectService.create(so); @@ -302,7 +304,7 @@ public void testAddDetachedRunToSequenceFile() throws IOException, InterruptedEx Path p = Files.createTempFile(null, null); Files.write(p, FASTQ_FILE_CONTENTS); - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); sf.setFile(p); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); Sample sample = sampleService.read(1L); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java index 115aa1f9080..2765f9a27ca 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java @@ -29,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -103,7 +104,7 @@ public void testGetSampleForProject() { public void testRemoveSequenceFileFromSample() { Sample s = new Sample(); s.setId(1111L); - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); sf.setId(2222L); SingleEndSequenceFile obj = new SingleEndSequenceFile(sf); obj.setId(2L); @@ -248,7 +249,7 @@ public void testGetCoverageForSampleSuccess() throws SequenceFileAnalysisExcepti Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new SequenceFile(); + SequenceFile sf1 = new LocalSequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -304,7 +305,7 @@ public void testGetTotalBasesForSampleSuccessOne() throws SequenceFileAnalysisEx Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new SequenceFile(); + SequenceFile sf1 = new LocalSequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -333,9 +334,9 @@ public void testGetTotalBasesForSampleSuccessTwo() throws SequenceFileAnalysisEx Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new SequenceFile(); + SequenceFile sf1 = new LocalSequenceFile(); sf1.setId(2222L); - SequenceFile sf2 = new SequenceFile(); + SequenceFile sf2 = new LocalSequenceFile(); sf1.setId(3333L); SampleSequencingObjectJoin join1 = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -368,7 +369,7 @@ public void testGetTotalBasesForSampleFailNoFastQC() throws SequenceFileAnalysis Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new SequenceFile(); + SequenceFile sf1 = new LocalSequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -389,7 +390,7 @@ public void testGetTotalBasesForSampleFailMultipleFastQC() throws SequenceFileAn Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new SequenceFile(); + SequenceFile sf1 = new LocalSequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -406,7 +407,7 @@ private Sample s(Long id) { } private SequenceFile sf(Long id) { - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); sf.setId(id); try { sf.setFile(Files.createTempFile(null, null)); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java index e89c0487f7b..fc6ee306da4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java @@ -16,7 +16,7 @@ import ca.corefacility.bioinformatics.irida.model.RemoteAPI; import ca.corefacility.bioinformatics.irida.model.remote.RemoteStatus; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.repositories.RemoteAPIRepository; import ca.corefacility.bioinformatics.irida.repositories.remote.SequenceFileRemoteRepository; @@ -45,7 +45,7 @@ public void testGetSequenceFilesForSample() { sample.setRemoteStatus(new RemoteStatus("http://nowhere", api)); - List filesList = Lists.newArrayList(new SingleEndSequenceFile(new SequenceFile())); + List filesList = Lists.newArrayList(new SingleEndSequenceFile(new LocalSequenceFile())); when(apiRepo.getRemoteAPIForUrl(seqFilesHref)).thenReturn(api); when(repository.list(seqFilesHref, api)).thenReturn(filesList); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java index e322d452ed6..618b578ca0b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java @@ -6,6 +6,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -17,7 +18,7 @@ */ public final class TestDataFactory { - /** + /** * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.User}. * * @return a {@link ca.corefacility.bioinformatics.irida.model.User} with identifier. @@ -51,7 +52,7 @@ public static Sample constructSample() { */ public static SequenceFile constructSequenceFile() throws IOException { Path f = Files.createTempFile(null, null); - SequenceFile sf = new SequenceFile(); + SequenceFile sf = new LocalSequenceFile(); sf.setId(1L); sf.setFile(f); return sf; From 28b419bc0c339f90efdd11a239c256df21f2aa15 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Sat, 28 Mar 2020 15:24:27 -0500 Subject: [PATCH 004/655] Refactored code --- .../processing/impl/FastqcFileProcessor.java | 4 +- .../IridaFileStorageServiceImpl.java | 44 +++++++++++-------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 58aa5978e1a..53925f772c9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -56,8 +56,6 @@ public class FastqcFileProcessor implements FileProcessor { private final MessageSource messageSource; private IridaFileStorageServiceImpl iridaFileStorageService; - private static final IridaFileStorageServiceImpl fileService = new IridaFileStorageServiceImpl(); - /** * Create a new {@link FastqcFileProcessor} * @@ -98,7 +96,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx LocaleContextHolder.getLocale())); try { uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - fileService.getTemporaryFile(fileToProcess)); + iridaFileStorageService.getTemporaryFile(fileToProcess)); BasicStats basicStats = new BasicStats(); PerBaseQualityScores pbqs = new PerBaseQualityScores(); PerSequenceQualityScores psqs = new PerSequenceQualityScores(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java index 98bbee7aa55..95c82fea8c7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java @@ -55,7 +55,7 @@ public IridaFileStorageServiceImpl(String storageType, String connectionStr, Str public File getTemporaryFile(Path file) { File fileToProcess = null; - if(storageType.equalsIgnoreCase("azure")) { + if(storageTypeIsAzure()) { // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(file.toAbsolutePath() .toString() @@ -75,7 +75,7 @@ public File getTemporaryFile(Path file) { } catch (BlobStorageException e) { logger.debug("Couldn't find file [" + e + "]"); } - } else if (storageType.equalsIgnoreCase("aws")) { + } else if (storageTypeIsAws()) { // Implement aws code to get file } else { fileToProcess = file.toFile(); @@ -89,7 +89,7 @@ public File getTemporaryFile(Path file) { @Override public Long getFileSize(Path file) { Long fileSize = 0L; - if(storageType.equalsIgnoreCase("azure")) { + if(storageTypeIsAzure()) { try { // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(file.toAbsolutePath() @@ -98,18 +98,10 @@ public Long getFileSize(Path file) { } catch (BlobStorageException e) { logger.debug("Couldn't calculate size as the file was not found [" + e + "]"); } - } else if (storageType.equalsIgnoreCase("aws")) { + } else { //implement aws code to get file size } - else { - try { - fileSize = Files.size(file); - } catch (NoSuchFileException e) { - logger.error("Could not find file " + file); - } catch (IOException e) { - logger.error("Could not calculate file size: ", e); - } - } + return fileSize; } @@ -118,7 +110,7 @@ public Long getFileSize(Path file) { */ @Override public void writeFile(Path source, Path target) { - if(storageType.equalsIgnoreCase("azure")) { + if(storageTypeIsAzure()) { // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(target.toAbsolutePath() .toString() @@ -127,7 +119,7 @@ public void writeFile(Path source, Path target) { logger.debug("Uploading file to azure: [" + target.getFileName() + "]"); blobClient.uploadFromFile(source.toString(), false); logger.debug("File uploaded to: [" + blobClient.getBlobUrl() + "]"); - } else if(storageType.equalsIgnoreCase("aws")){ + } else if(storageTypeIsAws()){ //implement aws code to upload file to s3 bucket } else { try { @@ -172,12 +164,28 @@ public boolean storageTypeIsLocal(){ return false; } + + public boolean storageTypeIsAzure(){ + if(storageType.equalsIgnoreCase("azure")){ + return true; + } + return false; + } + + public boolean storageTypeIsAws(){ + if(storageType.equalsIgnoreCase("aws")){ + return true; + } + return false; + } + + /** * {@inheritDoc} */ public String getFileName(Path file) { String fileName = ""; - if(storageType.equalsIgnoreCase("azure")) { + if(storageTypeIsAzure()) { blobClient = containerClient.getBlobClient(file.toAbsolutePath() .toString() .substring(1)); @@ -190,10 +198,8 @@ public String getFileName(Path file) { } catch (BlobStorageException e) { logger.debug("Couldn't find file [" + e + "]"); } - } else if(storageType.equalsIgnoreCase("aws")) { - //implement aws code to get file name from aws s3 bucket } else { - fileName = file.toFile().getName(); + //implement aws code to get file name from aws s3 bucket } return fileName; } From fbaa2cf5c39a286ec372b3538bff1d85211c342a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 31 Mar 2020 22:23:02 -0500 Subject: [PATCH 005/655] Created separate services for azure, aws, and local storage which the bean instantiates depending on storage type. --- .../services/IridaApiServicesConfig.java | 13 +- .../irida/model/project/ReferenceFile.java | 1 - .../model/sequenceFile/CloudSequenceFile.java | 19 +- .../model/sequenceFile/LocalSequenceFile.java | 14 +- .../model/sequenceFile/SequenceFile.java | 30 +-- .../workflow/analysis/AnalysisOutputFile.java | 1 - .../impl/ChecksumFileProcessor.java | 5 +- .../impl/DefaultFileProcessingChain.java | 7 +- .../processing/impl/FastqcFileProcessor.java | 7 +- .../processing/impl/GzipFileProcessor.java | 2 +- .../FilesystemSupplementedRepositoryImpl.java | 9 +- .../IridaFileStorageAwsServiceImpl.java | 108 +++++++++ .../IridaFileStorageAzureServiceImpl.java | 163 ++++++++++++++ .../IridaFileStorageLocalServiceImpl.java | 119 ++++++++++ .../filesystem}/IridaFileStorageService.java | 10 +- .../IridaFileStorageServiceImpl.java | 206 ------------------ .../impl/IridaFileStorageFactoryImpl.java | 6 +- .../20.05/add-sequence-file-type.xml | 21 ++ .../database/changesets/20.05/all-changes.xml | 2 + .../impl/unit/FastqcFileProcessorTest.java | 4 +- 20 files changed, 489 insertions(+), 258 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java rename src/main/java/ca/corefacility/bioinformatics/irida/{service => repositories/filesystem}/IridaFileStorageService.java (85%) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index d97135cb03f..d1b8f3039cb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -16,7 +16,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.impl.*; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.*; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionCleanupService; @@ -296,9 +296,14 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { } @Bean(name = "iridaFileStorageService") - public IridaFileStorageServiceImpl iridaFileStorageService() { - IridaFileStorageServiceImpl iridaFileStorageService = new IridaFileStorageServiceImpl(storageType, connectionStr, containerName); - return iridaFileStorageService; + public IridaFileStorageService iridaFileStorageService() { + if(storageType.equalsIgnoreCase("azure")) { + return new IridaFileStorageAzureServiceImpl(connectionStr, containerName); + } else if (storageType.equalsIgnoreCase("aws")) { + return new IridaFileStorageAwsServiceImpl(); + } else { + return new IridaFileStorageLocalServiceImpl(); + } } @Bean(name = "uploadFileProcessingChain") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index 05a1786b427..f88a1d57a15 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -26,7 +26,6 @@ import ca.corefacility.bioinformatics.irida.model.MutableIridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; /** * A reference file to be associated with a {@link Project}. diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java index 73d9ce70265..d1cae52a0de 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java @@ -2,35 +2,37 @@ import java.nio.file.Path; +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; import com.fasterxml.jackson.annotation.JsonIgnore; +@Entity +@DiscriminatorValue("cloud") public class CloudSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { private static final Logger logger = LoggerFactory.getLogger(CloudSequenceFile.class); - @Autowired - public IridaFileStorageServiceImpl iridaFileStorageService; - private Path file; public CloudSequenceFile() { super(); } public CloudSequenceFile(Path sampleFile) { super(sampleFile); - this.file = sampleFile; } @Override public String getLabel() { - return "testnewfile.txt"; + // Since the filesystem is virtual the path and filename are just one string + // so we split on the "/" and take the last token which is the file name + String [] cloudFileNameTokens = super.getFile().toString().split("/"); + return cloudFileNameTokens[cloudFileNameTokens.length-1]; } /** * Get the size of the file. @@ -41,7 +43,8 @@ public String getLabel() { @Override public String getFileSize() { String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageService.getFileSize(file), true); + // Need to implement cloud code to get file size + // size = IridaSequenceFile.humanReadableByteCount(6000, true); return size; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java index 5eee1425b7f..a7da2853da5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java @@ -5,29 +5,33 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; + import com.fasterxml.jackson.annotation.JsonIgnore; +@Entity +@DiscriminatorValue("local") public class LocalSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { private static final Logger logger = LoggerFactory.getLogger(LocalSequenceFile.class); - private Path file; public LocalSequenceFile() { super(); } public LocalSequenceFile(Path sampleFile) { super(sampleFile); - this.file = sampleFile; } @Override public String getLabel() { - return file.getFileName().toString(); + return super.getFile().getFileName().toString(); } /** * Get the size of the file. @@ -39,9 +43,9 @@ public String getLabel() { public String getFileSize() { String size = "N/A"; try { - size = IridaSequenceFile.humanReadableByteCount(Files.size(file), true); + size = IridaSequenceFile.humanReadableByteCount(Files.size(super.getFile()), true); } catch (NoSuchFileException e) { - logger.error("Could not find file " + file); + logger.error("Could not find file " + super.getFile()); } catch (IOException e) { logger.error("Could not calculate file size: ", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 73b90e538ac..e15f39bacbe 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -6,23 +6,7 @@ import java.util.Map; import java.util.Objects; -import javax.persistence.CascadeType; -import javax.persistence.CollectionTable; -import javax.persistence.Column; -import javax.persistence.ElementCollection; -import javax.persistence.Entity; -import javax.persistence.EntityListeners; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.MapKeyColumn; -import javax.persistence.OneToOne; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; -import javax.persistence.UniqueConstraint; +import javax.persistence.*; import javax.validation.constraints.NotNull; import org.hibernate.envers.Audited; @@ -43,7 +27,6 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -54,6 +37,11 @@ * particular {@link Sample}. */ @Entity +@Inheritance(strategy=InheritanceType.SINGLE_TABLE) +@DiscriminatorColumn( + name="sequence_file_type", + discriminatorType=DiscriminatorType.STRING +) @Table(name = "sequence_file") @Audited @EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class }) @@ -87,6 +75,8 @@ public abstract class SequenceFile extends IridaResourceSupport implements Mutab @Column(name = "file_revision_number") private Long fileRevisionNumber; // the filesystem file revision number + + // Key/value map of additional properties you could set on a sequence file. // This may contain optional sequencer specific properties. @ElementCollection(fetch = FetchType.EAGER) @@ -105,12 +95,16 @@ public abstract class SequenceFile extends IridaResourceSupport implements Mutab @JoinColumn(name = "remote_status") private RemoteStatus remoteStatus; + public SequenceFile() { createdDate = new Date(); fileRevisionNumber = 0L; optionalProperties = new HashMap<>(); } + public abstract String getLabel(); + public abstract String getFileSize(); + /** * Create a new {@link SequenceFile} with the given file Path * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index c248da8f2aa..0bb0bcac3bb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -26,7 +26,6 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; /** * Store file references to files produced by a workflow execution that we diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index a2aef3c2b53..e672aadb53b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -15,6 +15,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -27,6 +28,8 @@ public class ChecksumFileProcessor implements FileProcessor { private SequenceFileRepository fileRepository; + @Autowired + private IridaFileStorageService iridaFileStorageService; @Autowired public ChecksumFileProcessor(SequenceFileRepository fileRepository) { @@ -49,7 +52,7 @@ public void process(SequencingObject sequencingObject) { for (SequenceFile file : files) { - try (InputStream is = Files.newInputStream(file.getFile())) { + try (InputStream is = iridaFileStorageService.getFileInputStream(file)) { String shaDigest = DigestUtils.sha256Hex(is); logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java index b0157e323a1..e50b156e116 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import ca.corefacility.bioinformatics.irida.exceptions.FileProcessorTimeoutException; import ca.corefacility.bioinformatics.irida.model.sample.FileProcessorErrorQCEntry; @@ -13,6 +14,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessingChain; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -37,6 +39,9 @@ public class DefaultFileProcessingChain implements FileProcessingChain { private final SequencingObjectRepository sequencingObjectRepository; private QCEntryRepository qcRepository; + @Autowired + private IridaFileStorageService iridaFileStorageService; + public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, QCEntryRepository qcRepository, FileProcessor... fileProcessors) { this(sequencingObjectRepository, qcRepository, Arrays.asList(fileProcessors)); @@ -176,7 +181,7 @@ private SequencingObject getSettledSequencingObject(Long sequencingObjectId) thr if(sequencingObject.isPresent()) { Set files = sequencingObject.get().getFiles(); filesNotSettled = files.stream().anyMatch(f -> { - return !Files.exists(f.getFile()); + return iridaFileStorageService.fileExists(f.getFile()); }); } } while (filesNotSettled); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 53925f772c9..5da82f08018 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -9,9 +9,8 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; -import ca.corefacility.bioinformatics.irida.service.IridaFileStorageService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +53,7 @@ public class FastqcFileProcessor implements FileProcessor { private final SequenceFileRepository sequenceFileRepository; private final AnalysisOutputFileRepository outputFileRepository; private final MessageSource messageSource; - private IridaFileStorageServiceImpl iridaFileStorageService; + private IridaFileStorageService iridaFileStorageService; /** * Create a new {@link FastqcFileProcessor} @@ -66,7 +65,7 @@ public class FastqcFileProcessor implements FileProcessor { */ @Autowired public FastqcFileProcessor(final MessageSource messageSource, final SequenceFileRepository sequenceFileRepository, - AnalysisOutputFileRepository outputFileRepository, IridaFileStorageServiceImpl iridaFileStorageService) { + AnalysisOutputFileRepository outputFileRepository, IridaFileStorageService iridaFileStorageService) { this.messageSource = messageSource; this.sequenceFileRepository = sequenceFileRepository; this.outputFileRepository = outputFileRepository; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index a4a40966476..8550bbf60c8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -35,7 +35,7 @@ public class GzipFileProcessor implements FileProcessor { private static final String GZIP_EXTENSION = ".gz"; private final SequenceFileRepository sequenceFileRepository; - private boolean disableFileProcessor = false; + private boolean disableFileProcessor = true; private boolean removeCompressedFile; @Autowired diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index 06f1f698c3a..67dd825cf14 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -43,7 +43,7 @@ public abstract class FilesystemSupplementedRepositoryImpl pathFilter = f -> f.getType().equals(Path.class); // now find any members that are of type Path and shuffle them around: - Set pathFields = Arrays.stream(objectToWrite.getClass().getDeclaredFields()).filter(pathFilter) + Set pathFields = Arrays.stream(objectToWrite.getClass().getSuperclass().getDeclaredFields()).filter(pathFilter) .collect(Collectors.toSet()); + if(pathFields.size() == 0) { + pathFields = Arrays.stream(objectToWrite.getClass().getDeclaredFields()).filter(pathFilter) + .collect(Collectors.toSet()); + } + Set fieldsToUpdate = new HashSet<>(); for (Field field : pathFields) { ReflectionUtils.makeAccessible(field); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java new file mode 100644 index 00000000000..9f250d4d8f7 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -0,0 +1,108 @@ +package ca.corefacility.bioinformatics.irida.repositories.filesystem; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + +public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); + + //Aws Specific Variables + + + @Autowired + public IridaFileStorageAwsServiceImpl(){ + + } + + /** + * {@inheritDoc} + */ + @Override + public File getTemporaryFile(Path file) { + File fileToProcess = null; + + // Implement AWS code to get file + + return fileToProcess; + } + + /** + * {@inheritDoc} + */ + @Override + public Long getFileSize(Path file) { + Long fileSize = 0L; + // Implement AWS code to get file size + return fileSize; + } + + /** + * {@inheritDoc} + */ + @Override + public void writeFile(Path source, Path target) { + // Implement AWS code to upload file to bucket + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFiles() { + } + + @Override + public boolean storageTypeIsLocal(){ + return false; + } + + /** + * {@inheritDoc} + */ + public String getFileName(Path file) { + String fileName = ""; + // Implement AWS code to get file name + return fileName; + } + + @Override + public boolean fileExists(Path file) { + return false; + } + + @Override + public InputStream getFileInputStream(SequenceFile file) { + InputStream inputstream = null; + try { + inputstream = new FileInputStream(file.getFile().toString()); + + } catch(IOException e) { + + } + return inputstream; + } + +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java new file mode 100644 index 00000000000..b013c3b0469 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -0,0 +1,163 @@ +package ca.corefacility.bioinformatics.irida.repositories.filesystem; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Date; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobStorageException; + +@Service +public class IridaFileStorageAzureServiceImpl implements IridaFileStorageService { + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); + + private BlobServiceClient blobServiceClient; + private BlobContainerClient containerClient ; + private BlobClient blobClient; + + @Autowired + public IridaFileStorageAzureServiceImpl(String connectionStr, String containerName){ + this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) + .buildClient(); + this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + } + + /** + * {@inheritDoc} + */ + @Override + public File getTemporaryFile(Path file) { + File fileToProcess = null; + + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(file.toAbsolutePath() + .toString() + .substring(1)); + + try { + // Create a file that will be unique in the /tmp/ folder. We append the current date/time + // to the file name + String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. + String [] blobNameTokens = blobClient.getBlobName().split("/"); + String fileName = blobNameTokens[blobNameTokens.length-1]; + String filePath = tmpDir + fileName; + blobClient.downloadToFile(filePath); + fileToProcess = new File(filePath); + } catch (BlobStorageException e) { + logger.debug("Couldn't find file [" + e + "]"); + } + + return fileToProcess; + } + + /** + * {@inheritDoc} + */ + @Override + public Long getFileSize(Path file) { + Long fileSize = 0L; + try { + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(file.toAbsolutePath() + .toString().substring(1)); + fileSize = blobClient.getProperties().getBlobSize(); + } catch (BlobStorageException e) { + logger.debug("Couldn't calculate size as the file was not found [" + e + "]"); + } + return fileSize; + } + + /** + * {@inheritDoc} + */ + @Override + public void writeFile(Path source, Path target) { + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(target.toAbsolutePath() + .toString() + .substring(1)); + + logger.debug("Uploading file to azure: [" + target.getFileName() + "]"); + blobClient.uploadFromFile(source.toString(), false); + logger.debug("File uploaded to: [" + blobClient.getBlobUrl() + "]"); + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFiles() { + } + + @Override + public boolean storageTypeIsLocal(){ + return false; + } + + /** + * {@inheritDoc} + */ + public String getFileName(Path file) { + String fileName = ""; + blobClient = containerClient.getBlobClient(file.toAbsolutePath() + .toString() + .substring(1)); + try { + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. + String[] blobNameTokens = blobClient.getBlobName() + .split("/"); + fileName = blobNameTokens[blobNameTokens.length - 1]; + } catch (BlobStorageException e) { + logger.debug("Couldn't find file [" + e + "]"); + } + + return fileName; + } + + @Override + public boolean fileExists(Path file) { + blobClient = containerClient.getBlobClient(file.toAbsolutePath() + .toString() + .substring(1)); + if(blobClient.exists()) { + return false; + } + return true; + } + + @Override + public InputStream getFileInputStream(SequenceFile file) { + blobClient = containerClient.getBlobClient(file.getFile().toAbsolutePath() + .toString() + .substring(1)); + return blobClient.openInputStream(); + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java new file mode 100644 index 00000000000..ba300457b30 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -0,0 +1,119 @@ +package ca.corefacility.bioinformatics.irida.repositories.filesystem; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; + +public class IridaFileStorageLocalServiceImpl implements IridaFileStorageService{ + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalServiceImpl.class); + + @Autowired + public IridaFileStorageLocalServiceImpl(){ + } + + /** + * {@inheritDoc} + */ + @Override + public File getTemporaryFile(Path file) { + File fileToProcess = null; + + fileToProcess = file.toFile(); + + return fileToProcess; + } + + /** + * {@inheritDoc} + */ + @Override + public Long getFileSize(Path file) { + Long fileSize = 0L; + try { + fileSize = Files.size(file); + } catch (NoSuchFileException e) { + logger.error("Could not find file " + file); + } catch (IOException e) { + logger.error("Could not calculate file size: ", e); + } + + return fileSize; + } + + /** + * {@inheritDoc} + */ + @Override + public void writeFile(Path source, Path target) { + try { + Files.move(source, target); + logger.trace("Moved file " + source + " to " + target); + } catch (IOException e) { + logger.error("Unable to move file into new directory", e); + throw new StorageException("Failed to move file into new directory.", e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFiles() { + } + + /** + * {@inheritDoc} + */ + + public boolean storageTypeIsLocal(){ + return true; + } + + /** + * {@inheritDoc} + */ + public String getFileName(Path file) { + String fileName = ""; + fileName = file.getFileName().toString(); + return fileName; + } + + @Override + public boolean fileExists(Path file) { + return !Files.exists(file); + } + + @Override + public InputStream getFileInputStream(SequenceFile file) { + try { + return Files.newInputStream(file.getFile()); + } catch(IOException e) { + throw new FileProcessorException("could not calculate checksum", e); + } + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java similarity index 85% rename from src/main/java/ca/corefacility/bioinformatics/irida/service/IridaFileStorageService.java rename to src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 76a11660c74..0e7231a762a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -1,9 +1,12 @@ -package ca.corefacility.bioinformatics.irida.service; +package ca.corefacility.bioinformatics.irida.repositories.filesystem; import java.io.File; +import java.io.InputStream; import java.nio.file.Path; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + import com.azure.storage.blob.models.BlobStorageException; /** @@ -69,4 +72,9 @@ public interface IridaFileStorageService { * */ public void downloadFiles(); + + public boolean fileExists(Path file); + + public InputStream getFileInputStream(SequenceFile file); + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java deleted file mode 100644 index 95c82fea8c7..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceImpl.java +++ /dev/null @@ -1,206 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.filesystem; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.util.Date; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.service.IridaFileStorageService; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobStorageException; - -@Service -public class IridaFileStorageServiceImpl implements IridaFileStorageService { - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageServiceImpl.class); - - private String storageType; - - //Azure Specific Variables - private String containerName; - private String connectionStr; - private BlobServiceClient blobServiceClient; - private BlobContainerClient containerClient ; - private BlobClient blobClient; - - //AWS Specific Variables - - @Autowired - public IridaFileStorageServiceImpl(String storageType, String connectionStr, String containerName){ - this.storageType = storageType; - this.containerName = containerName; - this.connectionStr = connectionStr; - - this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) - .buildClient(); - this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - } - - /** - * {@inheritDoc} - */ - @Override - public File getTemporaryFile(Path file) { - File fileToProcess = null; - - if(storageTypeIsAzure()) { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString() - .substring(1)); - - try { - // Create a file that will be unique in the /tmp/ folder. We append the current date/time - // to the file name - String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String [] blobNameTokens = blobClient.getBlobName().split("/"); - String fileName = blobNameTokens[blobNameTokens.length-1]; - String filePath = tmpDir + fileName; - blobClient.downloadToFile(filePath); - fileToProcess = new File(filePath); - } catch (BlobStorageException e) { - logger.debug("Couldn't find file [" + e + "]"); - } - } else if (storageTypeIsAws()) { - // Implement aws code to get file - } else { - fileToProcess = file.toFile(); - } - return fileToProcess; - } - - /** - * {@inheritDoc} - */ - @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; - if(storageTypeIsAzure()) { - try { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString().substring(1)); - fileSize = blobClient.getProperties().getBlobSize(); - } catch (BlobStorageException e) { - logger.debug("Couldn't calculate size as the file was not found [" + e + "]"); - } - } else { - //implement aws code to get file size - } - - return fileSize; - } - - /** - * {@inheritDoc} - */ - @Override - public void writeFile(Path source, Path target) { - if(storageTypeIsAzure()) { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(target.toAbsolutePath() - .toString() - .substring(1)); - - logger.debug("Uploading file to azure: [" + target.getFileName() + "]"); - blobClient.uploadFromFile(source.toString(), false); - logger.debug("File uploaded to: [" + blobClient.getBlobUrl() + "]"); - } else if(storageTypeIsAws()){ - //implement aws code to upload file to s3 bucket - } else { - try { - Files.move(source, target); - logger.trace("Moved file " + source + " to " + target); - } catch (IOException e) { - logger.error("Unable to move file into new directory", e); - throw new StorageException("Failed to move file into new directory.", e); - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFiles() { - } - - /** - * {@inheritDoc} - */ - @Override - public boolean storageTypeIsLocal(){ - if(storageType.equalsIgnoreCase("local")){ - return true; - } - return false; - } - - - public boolean storageTypeIsAzure(){ - if(storageType.equalsIgnoreCase("azure")){ - return true; - } - return false; - } - - public boolean storageTypeIsAws(){ - if(storageType.equalsIgnoreCase("aws")){ - return true; - } - return false; - } - - - /** - * {@inheritDoc} - */ - public String getFileName(Path file) { - String fileName = ""; - if(storageTypeIsAzure()) { - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString() - .substring(1)); - try { - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String[] blobNameTokens = blobClient.getBlobName() - .split("/"); - fileName = blobNameTokens[blobNameTokens.length - 1]; - } catch (BlobStorageException e) { - logger.debug("Couldn't find file [" + e + "]"); - } - } else { - //implement aws code to get file name from aws s3 bucket - } - return fileName; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java index 826a37d8502..dcfc60cb901 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java @@ -7,18 +7,17 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; - import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; @Service public class IridaFileStorageFactoryImpl { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageFactoryImpl.class); @Autowired - private IridaFileStorageServiceImpl iridaFileStorageService; + private IridaFileStorageService iridaFileStorageService; @Autowired public IridaFileStorageFactoryImpl(){ @@ -35,6 +34,7 @@ public SequenceFile createSequenceFile(Path file) { public SequenceFile createEmptySequenceFile(){ if(iridaFileStorageService.storageTypeIsLocal()) { return new LocalSequenceFile(); + } else { return new CloudSequenceFile(); } diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml new file mode 100644 index 00000000000..6b73f0bd48c --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml index 0d0963ab0cc..99e1fccb844 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml @@ -6,4 +6,6 @@ http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd"> + diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index d9f6db10fcd..3a1ec2d5723 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -33,7 +33,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -45,7 +45,7 @@ public class FastqcFileProcessorTest { private FastqcFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private AnalysisOutputFileRepository outputFileRepository; - private IridaFileStorageServiceImpl iridaFileStorageService; + private IridaFileStorageService iridaFileStorageService; private MessageSource messageSource; private static final Logger logger = LoggerFactory.getLogger(FastqcFileProcessorTest.class); From 756694199b672acfee293001730141bb72a34488 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 1 Apr 2020 12:00:38 -0500 Subject: [PATCH 006/655] Updated file processors and tests to use iridafilestorageservice --- .../services/IridaApiServicesConfig.java | 4 ++-- .../impl/ChecksumFileProcessor.java | 2 +- .../impl/DefaultFileProcessingChain.java | 11 ++++----- .../processing/impl/GzipFileProcessor.java | 17 +++++++++----- .../IridaFileStorageAwsServiceImpl.java | 14 +++++++++-- .../IridaFileStorageAzureServiceImpl.java | 23 ++++++++++++++----- .../IridaFileStorageLocalServiceImpl.java | 22 +++++++++++++----- .../filesystem/IridaFileStorageService.java | 8 ++++++- .../unit/DefaultFileProcessingChainTest.java | 15 +++++++----- .../impl/unit/GzipFileProcessorTest.java | 6 +++-- 10 files changed, 84 insertions(+), 38 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index d1b8f3039cb..3d62b0d2c4e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -308,7 +308,7 @@ public IridaFileStorageService iridaFileStorageService() { @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, GzipFileProcessor gzipFileProcessor, + QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, GzipFileProcessor gzipFileProcessor, FastqcFileProcessor fastQcFileProcessor, ChecksumFileProcessor checksumProcessor, CoverageFileProcessor coverageProcessor, AutomatedAnalysisFileProcessor automatedAnalysisFileProcessor) { @@ -322,7 +322,7 @@ public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequenc fileProcessors.remove(gzipFileProcessor); } - return new DefaultFileProcessingChain(sequencingObjectRepository, qcRepository, fileProcessors); + return new DefaultFileProcessingChain(sequencingObjectRepository, qcRepository, iridaFileStorageService, fileProcessors); } @Bean(name = "fileProcessingChainExecutor") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index e672aadb53b..94f2110194b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -52,7 +52,7 @@ public void process(SequencingObject sequencingObject) { for (SequenceFile file : files) { - try (InputStream is = iridaFileStorageService.getFileInputStream(file)) { + try (InputStream is = iridaFileStorageService.getFileInputStream(file.getFile())) { String shaDigest = DigestUtils.sha256Hex(is); logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java index e50b156e116..e2a7e113c8a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java @@ -38,20 +38,19 @@ public class DefaultFileProcessingChain implements FileProcessingChain { private final SequencingObjectRepository sequencingObjectRepository; private QCEntryRepository qcRepository; - - @Autowired private IridaFileStorageService iridaFileStorageService; public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, FileProcessor... fileProcessors) { - this(sequencingObjectRepository, qcRepository, Arrays.asList(fileProcessors)); + QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, FileProcessor... fileProcessors) { + this(sequencingObjectRepository, qcRepository, iridaFileStorageService, Arrays.asList(fileProcessors)); } public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, List fileProcessors) { + QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, List fileProcessors) { this.fileProcessors = fileProcessors; this.sequencingObjectRepository = sequencingObjectRepository; this.qcRepository = qcRepository; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -181,7 +180,7 @@ private SequencingObject getSettledSequencingObject(Long sequencingObjectId) thr if(sequencingObject.isPresent()) { Set files = sequencingObject.get().getFiles(); filesNotSettled = files.stream().anyMatch(f -> { - return iridaFileStorageService.fileExists(f.getFile()); + return !iridaFileStorageService.fileExists(f.getFile()); }); } } while (filesNotSettled); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 8550bbf60c8..5fe81bce416 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -16,6 +16,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.util.FileUtils; @@ -35,18 +36,22 @@ public class GzipFileProcessor implements FileProcessor { private static final String GZIP_EXTENSION = ".gz"; private final SequenceFileRepository sequenceFileRepository; - private boolean disableFileProcessor = true; + private boolean disableFileProcessor = false; private boolean removeCompressedFile; + private IridaFileStorageService iridaFileStorageService; + @Autowired - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, IridaFileStorageService iridaFileStorageService) { this.sequenceFileRepository = sequenceFileRepository; removeCompressedFile = false; + this.iridaFileStorageService = iridaFileStorageService; } - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles, IridaFileStorageService iridaFileStorageService) { this.sequenceFileRepository = sequenceFileRepository; this.removeCompressedFile = removeCompressedFiles; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -99,7 +104,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc logger.debug("Not running processSingleFile. It has been disabled"); return; } - + Path file = sequenceFile.getFile(); String nameWithoutExtension = file.getFileName().toString(); @@ -110,10 +115,10 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); - if (FileUtils.isGzipped(file)) { + if (iridaFileStorageService.isGzipped(file)) { file = addExtensionToFilename(file, GZIP_EXTENSION); - try (GZIPInputStream zippedInputStream = new GZIPInputStream(Files.newInputStream(file))) { + try (GZIPInputStream zippedInputStream = new GZIPInputStream(iridaFileStorageService.getFileInputStream(file))) { logger.trace("Handling gzip compressed file."); Path targetDirectory = Files.createTempDirectory(null); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 9f250d4d8f7..8da9c25e2fd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.zip.GZIPInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,10 +95,10 @@ public boolean fileExists(Path file) { } @Override - public InputStream getFileInputStream(SequenceFile file) { + public InputStream getFileInputStream(Path file) { InputStream inputstream = null; try { - inputstream = new FileInputStream(file.getFile().toString()); + inputstream = new FileInputStream(file.toString()); } catch(IOException e) { @@ -105,4 +106,13 @@ public InputStream getFileInputStream(SequenceFile file) { return inputstream; } + @Override + public boolean isGzipped(Path file) throws IOException { + try (InputStream is = getFileInputStream(file)) { + byte[] bytes = new byte[2]; + is.read(bytes); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + } + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index b013c3b0469..4dca98dfcc8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -1,16 +1,17 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.Date; +import java.util.zip.GZIPInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -147,17 +148,27 @@ public boolean fileExists(Path file) { blobClient = containerClient.getBlobClient(file.toAbsolutePath() .toString() .substring(1)); - if(blobClient.exists()) { - return false; + if(blobClient.getProperties().getBlobSize() > 0) { + return true; } - return true; + return false; } @Override - public InputStream getFileInputStream(SequenceFile file) { - blobClient = containerClient.getBlobClient(file.getFile().toAbsolutePath() + public InputStream getFileInputStream(Path file) { + blobClient = containerClient.getBlobClient(file.toAbsolutePath() .toString() .substring(1)); return blobClient.openInputStream(); } + + @Override + public boolean isGzipped(Path file) throws IOException { + try (InputStream is = getFileInputStream(file)) { + byte[] bytes = new byte[2]; + is.read(bytes); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + } + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index ba300457b30..dce0f682fdc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -6,6 +6,8 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.zip.GZIPInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,9 +30,7 @@ public IridaFileStorageLocalServiceImpl(){ @Override public File getTemporaryFile(Path file) { File fileToProcess = null; - fileToProcess = file.toFile(); - return fileToProcess; } @@ -105,15 +105,25 @@ public String getFileName(Path file) { @Override public boolean fileExists(Path file) { - return !Files.exists(file); + return Files.exists(file); } @Override - public InputStream getFileInputStream(SequenceFile file) { + public InputStream getFileInputStream(Path file) { try { - return Files.newInputStream(file.getFile()); + return Files.newInputStream(file, StandardOpenOption.READ); } catch(IOException e) { - throw new FileProcessorException("could not calculate checksum", e); + throw new FileProcessorException("could not read file", e); + } + } + + @Override + public boolean isGzipped(Path file) throws IOException { + try (InputStream is = getFileInputStream(file)) { + byte[] bytes = new byte[2]; + is.read(bytes); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 0e7231a762a..6393282f324 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -1,8 +1,12 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.zip.GZIPInputStream; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -75,6 +79,8 @@ public interface IridaFileStorageService { public boolean fileExists(Path file); - public InputStream getFileInputStream(SequenceFile file); + public InputStream getFileInputStream(Path file); + + public boolean isGzipped(Path file) throws IOException; } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java index 3e76018557d..ba8adc740b0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.DefaultFileProcessingChain; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -38,6 +39,7 @@ public class DefaultFileProcessingChainTest { private SequencingObjectRepository objectRepository; private QCEntryRepository qcRepository; + private IridaFileStorageService iridaFileStorageService; private SequencingObject seqObject; private Long objectId = 1L; @@ -46,6 +48,7 @@ public class DefaultFileProcessingChainTest { public void setUp() { this.objectRepository = mock(SequencingObjectRepository.class); this.qcRepository = mock(QCEntryRepository.class); + this.iridaFileStorageService = mock(IridaFileStorageService.class); seqObject = new NoFileSequencingObject(); when(objectRepository.findById(objectId)).thenReturn(Optional.of(seqObject)); @@ -53,7 +56,7 @@ public void setUp() { @Test(expected = FileProcessorTimeoutException.class) public void testExceedsTimeout() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); fileProcessingChain.setTimeout(1); fileProcessingChain.setSleepDuration(0); @@ -62,7 +65,7 @@ public void testExceedsTimeout() throws FileProcessorTimeoutException { @Test public void testProcessEmptyChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); when(objectRepository.existsById(objectId)).thenReturn(true); fileProcessingChain.launchChain(objectId); @@ -70,7 +73,7 @@ public void testProcessEmptyChain() throws FileProcessorTimeoutException { @Test public void testFailWithContinueChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -84,7 +87,7 @@ public void testFailWithContinueChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFastFailProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -96,7 +99,7 @@ public void testFastFailProcessorChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFailOnProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -106,7 +109,7 @@ public void testFailOnProcessorChain() throws FileProcessorTimeoutException { @Test public void testFailWriteQCEntry() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 3b1fef73a21..d07f6fe7369 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -23,6 +23,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -35,12 +36,13 @@ public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageService); } @Test(expected = FileProcessorException.class) @@ -62,7 +64,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageService); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference From 2f7fb763df88f29a4fe6ffcd6e2aa18b28fdd605 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 12:09:47 -0500 Subject: [PATCH 007/655] Updated tests. Updated logic for sequencefile concatenation to work with the iridafilestoragefactory --- .../services/IridaApiServicesConfig.java | 8 +++ .../SequencingObjectConcatenatorFactory.java | 15 ++++-- .../impl/SequenceFilePairConcatenator.java | 7 ++- .../SingleEndSequenceFileConcatenator.java | 8 ++- .../impl/ChecksumFileProcessor.java | 5 +- .../AnalysisOutputFileRepositoryImpl.java | 5 +- .../GenomeAssemblyRepositoryImpl.java | 5 +- .../FilesystemSupplementedRepositoryImpl.java | 4 +- .../ReferenceFileRepositoryImpl.java | 5 +- .../SequenceFileRepositoryImpl.java | 5 +- .../web/samples/SamplesAjaxController.java | 7 ++- .../ria/web/samples/SamplesController.java | 5 +- .../impl/IridaFileStorageFactoryImpl.java | 8 +-- .../impl/SequencingObjectServiceImpl.java | 7 ++- .../sequencefile/SequenceFileResource.java | 3 -- .../RESTSampleSequenceFilesController.java | 5 +- ...quencingObjectConcatenatorFactoryTest.java | 28 +++++++--- .../SequenceFilePairConcatenatorTest.java | 15 +++++- ...SingleEndSequenceFileConcatenatorTest.java | 16 +++++- .../impl/unit/ChecksumFileProcessorTest.java | 15 +++++- .../impl/unit/FastqcFileProcessorTest.java | 22 +++++++- .../impl/unit/GzipFileProcessorTest.java | 11 ++++ .../SequenceFileRepositoryImplTest.java | 14 ++++- .../samples/SamplesAjaxControllerTest.java | 15 +++++- .../web/samples/SamplesControllerTest.java | 15 +++++- .../unit/SequencingObjectServiceTest.java | 15 +++++- .../SampleSequenceFilesControllerTest.java | 54 ++++++++++++++----- 27 files changed, 257 insertions(+), 65 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 3d62b0d2c4e..49f5003b8dc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -14,6 +14,8 @@ import ca.corefacility.bioinformatics.irida.plugins.IridaPluginException; import ca.corefacility.bioinformatics.irida.processing.FileProcessingChain; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; +import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; +import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; import ca.corefacility.bioinformatics.irida.processing.impl.*; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.*; @@ -22,6 +24,7 @@ import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionCleanupService; import ca.corefacility.bioinformatics.irida.service.TaxonomyService; import ca.corefacility.bioinformatics.irida.service.impl.InMemoryTaxonomyService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.AnalysisSubmissionCleanupServiceImpl; import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.util.IridaPluginMessageSource; @@ -306,6 +309,11 @@ public IridaFileStorageService iridaFileStorageService() { } } + @Bean(name = "iridaFileStorageFactory") + public IridaFileStorageFactoryImpl iridaFileStorageFactory(IridaFileStorageService iridaFileStorageService) { + return new IridaFileStorageFactoryImpl(iridaFileStorageService); + } + @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, GzipFileProcessor gzipFileProcessor, diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java index f4cd8472c63..e571bab70a3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java @@ -4,11 +4,16 @@ import java.util.Set; import java.util.stream.Collectors; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; /** * Factory class for returning an instance of @@ -24,13 +29,13 @@ public class SequencingObjectConcatenatorFactory { * @return the new {@link SequencingObjectConcatenator} */ @SuppressWarnings("unchecked") - public static SequencingObjectConcatenator getConcatenator(Class type) { + public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageFactoryImpl iridaFileStorageFactory) { // return the concatenator for the class if (type.equals(SingleEndSequenceFile.class)) { - return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(); + return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageFactory); } else if (type.equals(SequenceFilePair.class)) { - return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(); + return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageFactory); } else { throw new IllegalArgumentException("No concatenator exists for type " + type); } @@ -44,7 +49,7 @@ public static SequencingObjectConcatenator getCo * @return the new {@link SequencingObjectConcatenator} */ public static SequencingObjectConcatenator getConcatenator( - Collection objects) { + Collection objects, IridaFileStorageFactoryImpl iridaFileStorageFactory) { // get all the classes for the objects Set types = objects.stream().map(Object::getClass).collect(Collectors.toSet()); @@ -58,6 +63,6 @@ public static SequencingObjectConcatenator getConcat @SuppressWarnings("unchecked") Class type = (Class) types.iterator().next(); - return getConcatenator(type); + return getConcatenator(type, iridaFileStorageFactory); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 1a75817c3d6..c5e4306064e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -13,16 +13,19 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * {@link SequencingObjectConcatenator} for {@link SequenceFilePair}s */ +@Component public class SequenceFilePairConcatenator extends SequencingObjectConcatenator { - @Autowired private IridaFileStorageFactoryImpl iridaFileStorageFactory; - public SequenceFilePairConcatenator() { + @Autowired + public SequenceFilePairConcatenator(IridaFileStorageFactoryImpl iridaFileStorageFactory) { + this.iridaFileStorageFactory = iridaFileStorageFactory; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index b04d65f668b..532e4e28bd3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import java.io.IOException; @@ -13,16 +14,19 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * {@link SequenceFilePairConcatenator} for {@link SingleEndSequenceFile}s */ +@Component public class SingleEndSequenceFileConcatenator extends SequencingObjectConcatenator { - @Autowired private IridaFileStorageFactoryImpl iridaFileStorageFactory; - public SingleEndSequenceFileConcatenator() { + @Autowired + public SingleEndSequenceFileConcatenator(IridaFileStorageFactoryImpl iridaFileStorageFactory) { + this.iridaFileStorageFactory = iridaFileStorageFactory; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index 94f2110194b..e71893eb406 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -27,13 +27,12 @@ public class ChecksumFileProcessor implements FileProcessor { private static final Logger logger = LoggerFactory.getLogger(ChecksumFileProcessor.class); private SequenceFileRepository fileRepository; - - @Autowired private IridaFileStorageService iridaFileStorageService; @Autowired - public ChecksumFileProcessor(SequenceFileRepository fileRepository) { + public ChecksumFileProcessor(SequenceFileRepository fileRepository, IridaFileStorageService iridaFileStorageService) { this.fileRepository = fileRepository; + this.iridaFileStorageService = iridaFileStorageService; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java index 55db0862861..17aea89dfda 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java @@ -11,6 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** * Custom implementation of {@link FilesystemSupplementedRepositoryImpl} for @@ -23,8 +24,8 @@ public class AnalysisOutputFileRepositoryImpl extends FilesystemSupplementedRepo @Autowired public AnalysisOutputFileRepositoryImpl(EntityManager entityManager, - @Qualifier("outputFileBaseDirectory") Path baseDirectory) { - super(entityManager, baseDirectory); + @Qualifier("outputFileBaseDirectory") Path baseDirectory, IridaFileStorageService iridaFileStorageService) { + super(entityManager, baseDirectory, iridaFileStorageService); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java index 39fea994fa1..36cdd33342c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java @@ -11,6 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; import ca.corefacility.bioinformatics.irida.model.assembly.UploadedAssembly; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** * A {@link FilesystemSupplementedRepositoryImpl} implementation for {@link GenomeAssembly} @@ -20,8 +21,8 @@ public class GenomeAssemblyRepositoryImpl extends FilesystemSupplementedReposito @Autowired public GenomeAssemblyRepositoryImpl(EntityManager entityManager, - @Qualifier("assemblyFileBaseDirectory") Path baseDirectory) { - super(entityManager, baseDirectory); + @Qualifier("assemblyFileBaseDirectory") Path baseDirectory, IridaFileStorageService iridaFileStorageService) { + super(entityManager, baseDirectory, iridaFileStorageService); } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index 67dd825cf14..a6a35fa72af 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -42,12 +42,12 @@ public abstract class FilesystemSupplementedRepositoryImpl to throws ConcatenateException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory - .getConcatenator(toJoin); + .getConcatenator(toJoin, iridaFileStorageFactory); SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java index f9e79c62ec1..10d7a05d781 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java @@ -21,11 +21,8 @@ public class SequenceFileResource { private SequenceFile resource; - @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; public SequenceFileResource() { - resource = iridaFileStorageFactory.createEmptySequenceFile(); } public SequenceFileResource(SequenceFile sequenceFile) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 75999fd11dc..0c8270cfe9f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -135,8 +135,6 @@ public class RESTSampleSequenceFilesController { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; - - @Autowired private IridaFileStorageFactoryImpl iridaFileStorageFactory; protected RESTSampleSequenceFilesController() { @@ -144,11 +142,12 @@ protected RESTSampleSequenceFilesController() { @Autowired public RESTSampleSequenceFilesController(SampleService sampleService, SequencingRunService miseqRunService, - SequencingObjectService sequencingObjectService, AnalysisService analysisService) { + SequencingObjectService sequencingObjectService, AnalysisService analysisService, IridaFileStorageFactoryImpl iridaFileStorageFactory) { this.sampleService = sampleService; this.miseqRunService = miseqRunService; this.sequencingObjectService = sequencingObjectService; this.analysisService = analysisService; + this.iridaFileStorageFactory = iridaFileStorageFactory; } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index 96eb4f1893e..6aa4d4da3fd 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -1,12 +1,20 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; + import com.google.common.collect.Sets; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.util.Set; @@ -15,43 +23,51 @@ /** * Unit test for {@link SequencingObjectConcatenatorFactory} */ + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequencingObjectConcatenatorFactoryTest { + + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class); + SingleEndSequenceFile.class, iridaFileStorageFactory); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class); + SequenceFilePair.class, iridaFileStorageFactory); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageFactory); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageFactory); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageFactory); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageFactory); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 9086c28d77a..5e05f43b35c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -1,13 +1,20 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -16,6 +23,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequenceFilePairConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" @@ -23,9 +33,12 @@ public class SequenceFilePairConcatenatorTest { private SequenceFilePairConcatenator concat; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Before public void setUp() { - concat = new SequenceFilePairConcatenator(); + concat = new SequenceFilePairConcatenator(iridaFileStorageFactory); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 794f4323d36..52281b206f1 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -1,12 +1,20 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; + import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -15,6 +23,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SingleEndSequenceFileConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" @@ -22,9 +33,12 @@ public class SingleEndSequenceFileConcatenatorTest { SingleEndSequenceFileConcatenator concat; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Before public void setUp() { - concat = new SingleEndSequenceFileConcatenator(); + concat = new SingleEndSequenceFileConcatenator(iridaFileStorageFactory); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index 3bfdef81c91..8e37c174ecb 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -11,25 +11,38 @@ import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.ChecksumFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class ChecksumFileProcessorTest { private ChecksumFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; private static final String CHECKSUM = "aeaa0755dc44b393ffe12f02e9bd42b0169b12ca9c15708085db6a4ac9110ee0"; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new ChecksumFileProcessor(sequenceFileRepository); + fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 3a1ec2d5723..3a306993f82 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -16,15 +16,26 @@ import java.nio.file.Path; import java.util.Iterator; +import ca.corefacility.bioinformatics.irida.config.data.IridaApiJdbcDataSourceConfig; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; +import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import org.springframework.util.ReflectionUtils; import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; @@ -33,20 +44,29 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import com.github.springtestdbunit.DbUnitTestExecutionListener; + /** * Tests for {@link FastqcFileProcessor}. * * */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class FastqcFileProcessorTest { private FastqcFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private AnalysisOutputFileRepository outputFileRepository; - private IridaFileStorageService iridaFileStorageService; private MessageSource messageSource; + + @Autowired + private IridaFileStorageService iridaFileStorageService; + private static final Logger logger = LoggerFactory.getLogger(FastqcFileProcessorTest.class); private static final String SEQUENCE = "ACGTACGTN"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index d07f6fe7369..1d401933eb4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -16,8 +16,14 @@ import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -31,11 +37,16 @@ * * */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; + + @Autowired private IridaFileStorageService iridaFileStorageService; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index 0d37677d01b..f23d23cd48b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -18,7 +18,13 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepositoryImpl; @@ -28,6 +34,9 @@ * Tests for {@link FilesystemSupplementedRepositoryImpl}. * */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequenceFileRepositoryImplTest { private static final String TEMP_FILE_PREFIX = UUID.randomUUID().toString().replaceAll("-", ""); @@ -35,11 +44,14 @@ public class SequenceFileRepositoryImplTest { private Path baseDirectory; private EntityManager entityManager; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() throws IOException { baseDirectory = Files.createTempDirectory(TEMP_FILE_PREFIX); entityManager = mock(EntityManager.class); - repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory); + repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageService); } @After diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 9b667c25fad..043c05cf09a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -8,13 +8,19 @@ import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.web.multipart.MultipartHttpServletRequest; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; @@ -23,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.google.common.collect.ImmutableList; @@ -31,11 +38,17 @@ import static org.mockito.Matchers.eq; import static org.mockito.Mockito.*; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SamplesAjaxControllerTest { private SamplesAjaxController controller; private SequencingObjectService sequencingObjectService; private GenomeAssemblyService genomeAssemblyService; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + /* TEST DATA */ @@ -58,7 +71,7 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); + controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource, iridaFileStorageFactory); // Set up mocks when(sampleService.read(SAMPLE.getId())).thenReturn(SAMPLE); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index 3820d7ca3c7..49631500c70 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -11,15 +11,21 @@ import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.security.core.Authentication; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.ui.ExtendedModelMap; import org.springframework.ui.Model; import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.servlet.mvc.support.RedirectAttributesModelMap; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; import ca.corefacility.bioinformatics.irida.model.project.Project; @@ -33,6 +39,7 @@ import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -46,6 +53,9 @@ /** */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SamplesControllerTest { // Services @@ -59,6 +69,9 @@ public class SamplesControllerTest { private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Before public void setUp() { sampleService = mock(SampleService.class); @@ -70,7 +83,7 @@ public void setUp() { updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); controller = new SamplesController(sampleService, projectService, sequencingObjectService, - updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); + updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource, iridaFileStorageFactory); } // ************************************************************************************************ diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java index fd33deceee0..3203a7dad10 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.service.impl.unit; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun.LayoutType; import ca.corefacility.bioinformatics.irida.model.sample.Sample; @@ -11,10 +12,16 @@ import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.impl.SequencingObjectServiceImpl; import ca.corefacility.bioinformatics.irida.web.controller.test.unit.TestDataFactory; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import javax.validation.Validator; import java.io.IOException; @@ -22,6 +29,9 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequencingObjectServiceTest { SequencingObjectService service; @@ -31,6 +41,9 @@ public class SequencingObjectServiceTest { SequenceConcatenationRepository concatenationRepository; Validator validator; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Before public void setUp() { repository = mock(SequencingObjectRepository.class); @@ -40,7 +53,7 @@ public void setUp() { concatenationRepository = mock(SequenceConcatenationRepository.class); service = new SequencingObjectServiceImpl(repository, sequenceFileRepository, ssoRepository, - concatenationRepository, validator); + concatenationRepository, validator, iridaFileStorageFactory); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index 0211b67c24d..f6703a51bdb 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -16,15 +16,21 @@ import java.util.Iterator; import java.util.List; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.enums.SequencingRunUploadStatus; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.Matchers; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.hateoas.Link; import org.springframework.http.HttpStatus; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.ui.ModelMap; import org.springframework.util.FileCopyUtils; @@ -43,6 +49,7 @@ import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; +import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResourceCollection; import ca.corefacility.bioinformatics.irida.web.assembler.resource.RootResource; @@ -54,6 +61,9 @@ /** * Unit tests for {@link RESTSampleSequenceFilesController}. */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SampleSequenceFilesControllerTest { private RESTSampleSequenceFilesController controller; private SampleService sampleService; @@ -62,6 +72,9 @@ public class SampleSequenceFilesControllerTest { private AnalysisService analysisService; private SequencingRun sequencingRun; + @Autowired + private IridaFileStorageFactoryImpl iridaFileStorageFactory; + @Before public void setUp() { sampleService = mock(SampleService.class); @@ -70,7 +83,7 @@ public void setUp() { analysisService = mock(AnalysisService.class); sequencingRun = mock(SequencingRun.class); - controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService); + controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService, iridaFileStorageFactory); } @Test @@ -232,7 +245,9 @@ public void testAddNewSequenceFileToSample() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFileResource resource = new SequenceFileResource(); + SequenceFile emptySequenceFile = iridaFileStorageFactory.createEmptySequenceFile(); + + SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(6L); Path f = Files.createTempFile(null, null); MockMultipartFile mmf = new MockMultipartFile("filename", "filename", "blurgh", FileCopyUtils.copyToByteArray(f @@ -282,7 +297,9 @@ public void testAddNewSequenceFileToSampleCompletedRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFileResource resource = new SequenceFileResource(); + SequenceFile emptySequenceFile = iridaFileStorageFactory.createEmptySequenceFile(); + + SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(8L); Path f = Files.createTempFile(null, null); MockMultipartFile mmf = new MockMultipartFile("filename", "filename", "blurgh", FileCopyUtils.copyToByteArray(f @@ -306,7 +323,8 @@ public void testAddNewSequenceFileToSampleErrorRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFileResource resource = new SequenceFileResource(); + SequenceFile emptySequenceFile = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(8L); Path f = Files.createTempFile(null, null); MockMultipartFile mmf = new MockMultipartFile("filename", "filename", "blurgh", FileCopyUtils.copyToByteArray(f @@ -342,8 +360,11 @@ public void testAddNewSequenceFilePairToSample() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFileResource resource1 = new SequenceFileResource(); - SequenceFileResource resource2 = new SequenceFileResource(); + SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + + SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); + SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); resource1.setMiseqRunId(7L); resource2.setMiseqRunId(7L); Path f1 = Files.createTempFile(null, null); @@ -416,8 +437,11 @@ public void testAddNewSequenceFilePairToSampleMismatchedRunIDs() throws IOExcept SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFileResource resource1 = new SequenceFileResource(); - SequenceFileResource resource2 = new SequenceFileResource(); + SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + + SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); + SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); resource1.setMiseqRunId(1L); resource2.setMiseqRunId(2L); Path f1 = Files.createTempFile(null, null); @@ -442,8 +466,11 @@ public void testAddNewSequenceFilePairToSampleCompletedRun() throws IOException SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFileResource resource1 = new SequenceFileResource(); - SequenceFileResource resource2 = new SequenceFileResource(); + SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + + SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); + SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); resource1.setMiseqRunId(4L); resource2.setMiseqRunId(4L); Path f1 = Files.createTempFile(null, null); @@ -472,8 +499,11 @@ public void testAddNewSequenceFilePairToSampleErrorRun() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFileResource resource1 = new SequenceFileResource(); - SequenceFileResource resource2 = new SequenceFileResource(); + SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + + SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); + SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); resource1.setMiseqRunId(4L); resource2.setMiseqRunId(4L); Path f1 = Files.createTempFile(null, null); From 457d4f90bf21d9492941f00c6a6c3d49c8d08aca Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 13:31:17 -0500 Subject: [PATCH 008/655] Set default value to local for storage type if not set in irida.conf or argument is not found. Removed controller methods which are now in the samplesajaxcontroller --- .../services/IridaApiServicesConfig.java | 2 +- .../ria/web/samples/SamplesController.java | 47 ------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 49f5003b8dc..9ace7bf3333 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -131,7 +131,7 @@ public class IridaApiServicesConfig { @Value("${locales.enabled}") private String availableLocales; - @Value("${irida.storage.type}") + @Value("${irida.storage.type:local}") private String storageType; @Value("${azure.container.name}") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 6284ac1e7b6..0616eaaf2b3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -589,53 +589,6 @@ public String concatenateSequenceFiles(@PathVariable Long sampleId, @RequestPara return "redirect:" + redirectUrl; } - /** - * Create a {@link SequenceFile} and add it to a {@link Sample} - * - * @param file - * {@link MultipartFile} - * @param sample - * {@link Sample} to add the file to. - * @throws IOException - */ - private void createSequenceFileInSample(MultipartFile file, Sample sample) throws IOException { - SequenceFile sequenceFile = createSequenceFile(file); - sequencingObjectService.createSequencingObjectInSample(new SingleEndSequenceFile(sequenceFile), sample); - } - - /** - * Create {@link SequenceFile}'s then add them as {@link SequenceFilePair} - * to a {@link Sample} - * - * @param pair - * {@link List} of {@link MultipartFile} - * @param sample - * {@link Sample} to add the pair to. - * @throws IOException - */ - private void createSequenceFilePairsInSample(List pair, Sample sample) throws IOException { - SequenceFile firstFile = createSequenceFile(pair.get(0)); - SequenceFile secondFile = createSequenceFile(pair.get(1)); - sequencingObjectService.createSequencingObjectInSample(new SequenceFilePair(firstFile, secondFile), sample); - } - - /** - * Private method to move the sequence file into the correct directory and - * create the {@link SequenceFile} object. - * - * @param file - * {@link MultipartFile} sequence file uploaded. - * - * @return {@link SequenceFile} - * @throws IOException - * Exception thrown if there is an error handling the file. - */ - private SequenceFile createSequenceFile(MultipartFile file) throws IOException { - Path temp = Files.createTempDirectory(null); - Path target = temp.resolve(file.getOriginalFilename()); - file.transferTo(target.toFile()); - return iridaFileStorageFactory.createSequenceFile(target); - } /** * Test if the {@link User} is a {@link ProjectRole#PROJECT_OWNER} for the From 4d9c84400b1fe6c66e785ceb7ccdf1ffed3150dc Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 13:39:47 -0500 Subject: [PATCH 009/655] Moved default setting of irida.storage.type to filesystem.properties --- .../irida/config/services/IridaApiServicesConfig.java | 2 +- .../bioinformatics/irida/config/filesystem.properties | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 9ace7bf3333..49f5003b8dc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -131,7 +131,7 @@ public class IridaApiServicesConfig { @Value("${locales.enabled}") private String availableLocales; - @Value("${irida.storage.type:local}") + @Value("${irida.storage.type}") private String storageType; @Value("${azure.container.name}") diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties index a2036e1082d..90b66d7f53e 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties @@ -11,3 +11,5 @@ file.processing.queue.capacity=512 file.processing.process=true file.upload.max_size=21474836480 + +irida.storage.type=local \ No newline at end of file From 55650284aa1fd06b1be2707159236feaa0a1eea5 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 14:03:43 -0500 Subject: [PATCH 010/655] Removed unused imports and added missing javadoc comments --- .../irida/config/services/IridaApiServicesConfig.java | 2 -- .../irida/model/sequenceFile/CloudSequenceFile.java | 5 +++++ .../irida/model/sequenceFile/LocalSequenceFile.java | 5 +++++ .../irida/model/sequenceFile/SequenceFile.java | 11 +++++++++++ .../SequencingObjectConcatenatorFactory.java | 6 ++---- .../impl/SingleEndSequenceFileConcatenator.java | 1 - .../irida/processing/impl/ChecksumFileProcessor.java | 1 - .../processing/impl/DefaultFileProcessingChain.java | 2 -- .../irida/processing/impl/GzipFileProcessor.java | 1 - .../FilesystemSupplementedRepositoryImpl.java | 1 - .../filesystem/IridaFileStorageAwsServiceImpl.java | 7 +++++-- .../filesystem/IridaFileStorageAzureServiceImpl.java | 8 +++++--- .../filesystem/IridaFileStorageLocalServiceImpl.java | 6 +++++- .../filesystem/IridaFileStorageService.java | 8 -------- .../irida/ria/web/samples/SamplesController.java | 2 -- .../service/impl/IridaFileStorageFactoryImpl.java | 6 ++++-- .../resource/sequencefile/SequenceFileResource.java | 3 --- 17 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 49f5003b8dc..f2844614a55 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -14,8 +14,6 @@ import ca.corefacility.bioinformatics.irida.plugins.IridaPluginException; import ca.corefacility.bioinformatics.irida.processing.FileProcessingChain; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; -import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; -import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; import ca.corefacility.bioinformatics.irida.processing.impl.*; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.*; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java index d1cae52a0de..38c9e9ae320 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java @@ -10,9 +10,14 @@ import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sample.Sample; import com.fasterxml.jackson.annotation.JsonIgnore; +/** + * A file that may be stored somewhere on a cloud file system and belongs to a + * particular {@link Sample}. + */ @Entity @DiscriminatorValue("cloud") public class CloudSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java index a7da2853da5..24071435a82 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java @@ -13,9 +13,14 @@ import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sample.Sample; import com.fasterxml.jackson.annotation.JsonIgnore; +/** + * A file that may be stored somewhere on the local file system and belongs to a + * particular {@link Sample}. + */ @Entity @DiscriminatorValue("local") public class LocalSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index e15f39bacbe..bf67914248b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -102,7 +102,18 @@ public SequenceFile() { optionalProperties = new HashMap<>(); } + /** + * Get the implementation-specific file label. + * + * @return the file label. + */ public abstract String getLabel(); + + /** + * Get the implementation-specific file size. + * + * @return the file size. + */ public abstract String getFileSize(); /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java index e571bab70a3..cb4c6fa768e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java @@ -4,15 +4,11 @@ import java.util.Set; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; /** @@ -26,6 +22,7 @@ public class SequencingObjectConcatenatorFactory { * * @param type the class to get a concatenator for * @param The type this concatenator should act on + * @param iridaFileStorageFactory The file storage factory component * @return the new {@link SequencingObjectConcatenator} */ @SuppressWarnings("unchecked") @@ -46,6 +43,7 @@ public static SequencingObjectConcatenator getCo * {@link SequencingObject}s * * @param objects the {@link SequencingObject}s to get the concatenator for + * @param iridaFileStorageFactory The file storage factory component * @return the new {@link SequencingObjectConcatenator} */ public static SequencingObjectConcatenator getConcatenator( diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 532e4e28bd3..ceeeb462521 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,7 +5,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import java.io.IOException; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index e71893eb406..15cbc9b4cad 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; import java.util.Set; import org.apache.commons.codec.digest.DigestUtils; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java index e2a7e113c8a..f39a4a99891 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java @@ -1,11 +1,9 @@ package ca.corefacility.bioinformatics.irida.processing.impl; -import java.nio.file.Files; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import ca.corefacility.bioinformatics.irida.exceptions.FileProcessorTimeoutException; import ca.corefacility.bioinformatics.irida.model.sample.FileProcessorErrorQCEntry; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 5fe81bce416..006bf5f749e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -18,7 +18,6 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -import ca.corefacility.bioinformatics.irida.util.FileUtils; /** * Handle gzip-ed files (if necessary). This class partially assumes that gzip diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index a6a35fa72af..24d8cd6afab 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -20,7 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.ReflectionUtils; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 8da9c25e2fd..e7226f5ff3a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -10,9 +10,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; - +/** + * Component implementation of file utitlities for aws storage + */ +@Component public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 4dca98dfcc8..5d6671547c7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -10,8 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - +import org.springframework.stereotype.Component; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -19,7 +18,10 @@ import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobStorageException; -@Service +/** + * Component implementation of file utitlities for azure storage + */ +@Component public class IridaFileStorageAzureServiceImpl implements IridaFileStorageService { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index dce0f682fdc..7c2b9a033ba 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -12,11 +12,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +/** + * Component implementation of file utitlities for local storage + */ +@Component public class IridaFileStorageLocalServiceImpl implements IridaFileStorageService{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalServiceImpl.class); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 6393282f324..173d1280933 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -3,15 +3,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.zip.GZIPInputStream; - -import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; - -import com.azure.storage.blob.models.BlobStorageException; /** * Interface describing methods for performing storage actions diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 0616eaaf2b3..708f40c506c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.servlet.mvc.support.RedirectAttributes; @@ -36,7 +35,6 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java index 824af94f679..dfec2f07fc4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java @@ -5,7 +5,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; @@ -13,7 +12,10 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; -@Component +/** + * Service implementation for creating SequenceFiles {@link LocalSequenceFile} and {@link CloudSequenceFile} + */ +@Service public class IridaFileStorageFactoryImpl { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageFactoryImpl.class); private IridaFileStorageService iridaFileStorageService; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java index 10d7a05d781..9376b031282 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java @@ -1,9 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.assembler.resource.sequencefile; -import org.springframework.beans.factory.annotation.Autowired; - import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; From 8c712e4f2ee26b51a23afa3617fea986d1791822 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 14:10:18 -0500 Subject: [PATCH 011/655] Set default values for azure container and connection string if not found --- .../irida/config/services/IridaApiServicesConfig.java | 4 ++-- .../bioinformatics/irida/config/filesystem.properties | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index f2844614a55..7b81be31f9c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -132,10 +132,10 @@ public class IridaApiServicesConfig { @Value("${irida.storage.type}") private String storageType; - @Value("${azure.container.name}") + @Value("${azure.container.name:#{null}}") private String containerName; - @Value("${azure.account.connection.string}") + @Value("${azure.account.connection.string:#{null}}") private String connectionStr; diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties index 90b66d7f53e..1b8a047b91f 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties @@ -12,4 +12,4 @@ file.processing.process=true file.upload.max_size=21474836480 -irida.storage.type=local \ No newline at end of file +irida.storage.type=local From e9e990a2d6e2d3322ca67dc763c809e0532f83ec Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 14:39:24 -0500 Subject: [PATCH 012/655] Added missing javadoc comments --- .../filesystem/IridaFileStorageService.java | 21 +++++++++++++++++ .../impl/IridaFileStorageFactoryImpl.java | 23 +++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 173d1280933..3427dce9fd4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -69,10 +69,31 @@ public interface IridaFileStorageService { */ public void downloadFiles(); + /** + * Checks if file exists + * + * @param file The path to the file + * @return true if file exists otherwise false + * + */ public boolean fileExists(Path file); + /** + * Gets the file inputstream + * + * @param file The path to the file + * @return file inputstream + * + */ public InputStream getFileInputStream(Path file); + /** + * Checks if file is gzipped + * + * @param file The path to the file + * @return true if file is gzipped otherwise false + * + */ public boolean isGzipped(Path file) throws IOException; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java index dfec2f07fc4..721b7fa768a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java @@ -21,20 +21,35 @@ public class IridaFileStorageFactoryImpl { private IridaFileStorageService iridaFileStorageService; @Autowired - public IridaFileStorageFactoryImpl(IridaFileStorageService iridaFileStorageService){ + public IridaFileStorageFactoryImpl(IridaFileStorageService iridaFileStorageService) { this.iridaFileStorageService = iridaFileStorageService; } + /** + * Creates a {@link LocalSequenceFile} or {@link CloudSequenceFile} depending + * on the storage type. + * + * @param file The path to the file for which to create an object for + * @return a {@link LocalSequenceFile} or {@link CloudSequenceFile} as + * a {@link SequenceFile} + */ public SequenceFile createSequenceFile(Path file) { - if(iridaFileStorageService.storageTypeIsLocal()) { + if (iridaFileStorageService.storageTypeIsLocal()) { return new LocalSequenceFile(file); } else { return new CloudSequenceFile(file); } } - public SequenceFile createEmptySequenceFile(){ - if(iridaFileStorageService.storageTypeIsLocal()) { + /** + * Creates an empty {@link LocalSequenceFile} or {@link CloudSequenceFile} depending + * on the storage type. + * + * @return an empty {@link LocalSequenceFile} or {@link CloudSequenceFile} object as + * a {@link SequenceFile} + */ + public SequenceFile createEmptySequenceFile() { + if (iridaFileStorageService.storageTypeIsLocal()) { return new LocalSequenceFile(); } else { From 3a5f7308fa16c27f4cd9bc0a4ca57099b0e65078 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 15:11:59 -0500 Subject: [PATCH 013/655] Added missing @param and @throws --- .../irida/processing/impl/FastqcFileProcessor.java | 1 + .../irida/repositories/filesystem/IridaFileStorageService.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 5da82f08018..00acf3a28e1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -62,6 +62,7 @@ public class FastqcFileProcessor implements FileProcessor { * analysis). * @param sequenceFileRepository Repository for storing sequence files * @param outputFileRepository Repository for storing analysis output files + * @param iridaFileStorageService The irida file storage service */ @Autowired public FastqcFileProcessor(final MessageSource messageSource, final SequenceFileRepository sequenceFileRepository, diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 3427dce9fd4..f6879388ed7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -92,6 +92,7 @@ public interface IridaFileStorageService { * * @param file The path to the file * @return true if file is gzipped otherwise false + * @throws {@link IOException} if file can't be read * */ public boolean isGzipped(Path file) throws IOException; From 3d0dc9cae2051bb71b000f87e60060e6ff5dd0f6 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 15:30:43 -0500 Subject: [PATCH 014/655] Fixed @throws text --- .../filesystem/IridaFileStorageLocalServiceImpl.java | 1 - .../irida/repositories/filesystem/IridaFileStorageService.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index 7c2b9a033ba..990d4c129a2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -93,7 +93,6 @@ public void downloadFiles() { /** * {@inheritDoc} */ - public boolean storageTypeIsLocal(){ return true; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index f6879388ed7..634f3bb1396 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -92,7 +92,7 @@ public interface IridaFileStorageService { * * @param file The path to the file * @return true if file is gzipped otherwise false - * @throws {@link IOException} if file can't be read + * @throws IOException if file can't be read * */ public boolean isGzipped(Path file) throws IOException; From ba008ae0e244fc7c938a58e9c9988c16e95e1060 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Apr 2020 15:44:56 -0500 Subject: [PATCH 015/655] Removed iridafilestoragefactory from samplescontroller and samplescontrollertest as the code for which it was used has been moved to samplesajaxcontroller --- .../irida/ria/web/samples/SamplesController.java | 5 +---- .../unit/web/samples/SamplesControllerTest.java | 14 +------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 708f40c506c..dcf06e75b08 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -42,7 +42,6 @@ import ca.corefacility.bioinformatics.irida.ria.web.BaseController; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleDetails; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; @@ -111,13 +110,12 @@ public class SamplesController extends BaseController { private final UpdateSamplePermission updateSamplePermission; private final MessageSource messageSource; - private IridaFileStorageFactoryImpl iridaFileStorageFactory; @Autowired public SamplesController(SampleService sampleService, ProjectService projectService, SequencingObjectService sequencingObjectService, UpdateSamplePermission updateSamplePermission, MetadataTemplateService metadataTemplateService, GenomeAssemblyService genomeAssemblyService, - MessageSource messageSource, IridaFileStorageFactoryImpl iridaFileStorageFactory) { + MessageSource messageSource) { this.sampleService = sampleService; this.projectService = projectService; this.sequencingObjectService = sequencingObjectService; @@ -125,7 +123,6 @@ public SamplesController(SampleService sampleService, ProjectService projectServ this.metadataTemplateService = metadataTemplateService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; - this.iridaFileStorageFactory = iridaFileStorageFactory; } /************************************************************************************************ diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index 49631500c70..ace2ef143bc 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -11,21 +11,15 @@ import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.security.core.Authentication; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.ui.ExtendedModelMap; import org.springframework.ui.Model; import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.servlet.mvc.support.RedirectAttributesModelMap; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; import ca.corefacility.bioinformatics.irida.model.project.Project; @@ -39,7 +33,6 @@ import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -53,9 +46,6 @@ /** */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SamplesControllerTest { // Services @@ -69,8 +59,6 @@ public class SamplesControllerTest { private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; - @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; @Before public void setUp() { @@ -83,7 +71,7 @@ public void setUp() { updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); controller = new SamplesController(sampleService, projectService, sequencingObjectService, - updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource, iridaFileStorageFactory); + updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); } // ************************************************************************************************ From 8dcfd9491da7f6605d7e5b284573e3a583776de9 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 9 Apr 2020 21:59:43 -0500 Subject: [PATCH 016/655] IridaFileStorageService is now wired into the sequencefile entity using @postload --- .../model/sequenceFile/CloudSequenceFile.java | 3 ++- .../model/sequenceFile/LocalSequenceFile.java | 8 +----- .../model/sequenceFile/SequenceFile.java | 15 ++++++++++- .../IridaFileStorageServiceListener.java | 26 +++++++++++++++++++ 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java index 38c9e9ae320..5657266185e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java @@ -48,8 +48,9 @@ public String getLabel() { @Override public String getFileSize() { String size = "N/A"; + super.getIridaFileStorageService().getFileSize(super.getFile()); // Need to implement cloud code to get file size - // size = IridaSequenceFile.humanReadableByteCount(6000, true); + size = IridaSequenceFile.humanReadableByteCount(super.getIridaFileStorageService().getFileSize(super.getFile()), true); return size; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java index 24071435a82..f895b2ef48d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java @@ -47,13 +47,7 @@ public String getLabel() { @Override public String getFileSize() { String size = "N/A"; - try { - size = IridaSequenceFile.humanReadableByteCount(Files.size(super.getFile()), true); - } catch (NoSuchFileException e) { - logger.error("Could not find file " + super.getFile()); - } catch (IOException e) { - logger.error("Could not calculate file size: ", e); - } + size = IridaSequenceFile.humanReadableByteCount(super.getIridaFileStorageService().getFileSize(super.getFile()), true); return size; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index bf67914248b..e1c898a6a99 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -27,6 +27,8 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceListener; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -44,12 +46,14 @@ ) @Table(name = "sequence_file") @Audited -@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class }) +@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageServiceListener.class }) public abstract class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, VersionedFileFields, IridaSequenceFile, RemoteSynchronizable { private static final Logger logger = LoggerFactory.getLogger(SequenceFile.class); + private static IridaFileStorageService iridaFileStorageService; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -287,4 +291,13 @@ public String getUploadSha256() { public void setUploadSha256(String uploadSha256) { this.uploadSha256 = uploadSha256; } + + public IridaFileStorageService getIridaFileStorageService() { + return iridaFileStorageService; + } + + public void setIridaFileStorageService(IridaFileStorageService iridaFileStorageService) { + this.iridaFileStorageService = iridaFileStorageService; + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java new file mode 100644 index 00000000000..a69ba8fd125 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java @@ -0,0 +1,26 @@ +package ca.corefacility.bioinformatics.irida.repositories.filesystem; + +import javax.persistence.PostLoad; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + +@Component +public class IridaFileStorageServiceListener { + private final Logger logger = LoggerFactory.getLogger(IridaFileStorageServiceListener.class); + + @Autowired + private IridaFileStorageService iridaFileStorageService; + + @PostLoad + public void afterEntityLoad(SequenceFile sequenceFile) { + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); + iridaFileStorageService.fileExists(sequenceFile.getFile()); + sequenceFile.setIridaFileStorageService(iridaFileStorageService); + } +} \ No newline at end of file From e9d5d85b90d82ec70a6779780f6b0d0fd138cd82 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Sat, 11 Apr 2020 16:31:05 -0500 Subject: [PATCH 017/655] Removed discriminator column and discriminator values for sequencefile, localsequencefile, and cloud sequencefile and added a DTYPE to th the sequencefile table. Added iridafilestorageservice entity listener with provides the SequenceFile model with access to the iridiaFileStorageService after it has been loaded. Moved directory creation when using local storage into the iridafilestorage implementation for the local storage. Updated ProjectReferenceFileController to get the size of the reference file from the iridaFileStorageService --- .../model/sequenceFile/CloudSequenceFile.java | 1 - .../model/sequenceFile/LocalSequenceFile.java | 1 - .../model/sequenceFile/SequenceFile.java | 7 ++----- .../IridaFileStorageServiceListener.java | 15 ++++++++++--- .../FilesystemSupplementedRepositoryImpl.java | 21 ++----------------- .../IridaFileStorageAwsServiceImpl.java | 2 +- .../IridaFileStorageAzureServiceImpl.java | 2 +- .../IridaFileStorageLocalServiceImpl.java | 17 ++++++++++++++- .../filesystem/IridaFileStorageService.java | 3 +-- .../ProjectReferenceFileController.java | 12 +++++------ ...ype.xml => add-dtype-to-sequence-file.xml} | 14 ++++++------- .../database/changesets/20.05/all-changes.xml | 3 ++- 12 files changed, 48 insertions(+), 50 deletions(-) rename src/main/java/ca/corefacility/bioinformatics/irida/repositories/{filesystem => entity/listeners}/IridaFileStorageServiceListener.java (57%) rename src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/{add-sequence-file-type.xml => add-dtype-to-sequence-file.xml} (56%) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java index 5657266185e..8b3de03e8c8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java @@ -19,7 +19,6 @@ * particular {@link Sample}. */ @Entity -@DiscriminatorValue("cloud") public class CloudSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { private static final Logger logger = LoggerFactory.getLogger(CloudSequenceFile.class); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java index f895b2ef48d..c22a35fab7f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java @@ -22,7 +22,6 @@ * particular {@link Sample}. */ @Entity -@DiscriminatorValue("local") public class LocalSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { private static final Logger logger = LoggerFactory.getLogger(LocalSequenceFile.class); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index e1c898a6a99..1cfda2e7b0f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -26,9 +26,10 @@ import ca.corefacility.bioinformatics.irida.model.remote.RemoteSynchronizable; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageServiceListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageServiceListener; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -40,10 +41,6 @@ */ @Entity @Inheritance(strategy=InheritanceType.SINGLE_TABLE) -@DiscriminatorColumn( - name="sequence_file_type", - discriminatorType=DiscriminatorType.STRING -) @Table(name = "sequence_file") @Audited @EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageServiceListener.class }) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageServiceListener.java similarity index 57% rename from src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java rename to src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageServiceListener.java index a69ba8fd125..e9e3e733cc5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageServiceListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageServiceListener.java @@ -1,4 +1,4 @@ -package ca.corefacility.bioinformatics.irida.repositories.filesystem; +package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; import javax.persistence.PostLoad; @@ -9,7 +9,11 @@ import org.springframework.web.context.support.SpringBeanAutowiringSupport; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +/** + * Component implementation to run on an entity after it is has been accessed from the db. + */ @Component public class IridaFileStorageServiceListener { private final Logger logger = LoggerFactory.getLogger(IridaFileStorageServiceListener.class); @@ -17,10 +21,15 @@ public class IridaFileStorageServiceListener { @Autowired private IridaFileStorageService iridaFileStorageService; + /** + * After the SequenceFile entity is loaded this method will provide + * the entity access to the iridaFileStorageService + * + * @param sequenceFile The entity to provide the iridaFileStorageService to + */ @PostLoad - public void afterEntityLoad(SequenceFile sequenceFile) { + public void afterSequenceFileLoad(SequenceFile sequenceFile) { SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); - iridaFileStorageService.fileExists(sequenceFile.getFile()); sequenceFile.setIridaFileStorageService(iridaFileStorageService); } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index 24d8cd6afab..48bbd1ce3e8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -207,7 +207,7 @@ public Type saveMetadata(final Type entity) { * works using reflection to automagically find and update any internal * {@link Path} members on the {@link VersionedFileFields}. This class * **does not** update the object in the database - * + * * @param baseDirectory * @param iridaThing */ @@ -248,24 +248,7 @@ private Type writeFilesToDisk(Path baseDirectory, Type objectToWrite) { Path source = (Path) ReflectionUtils.getField(field, objectToWrite); Path target = sequenceFileDirWithRevision.resolve(source.getFileName()); logger.debug("Target is [" + target.toString() + "]"); - - if(iridaFileStorageService.storageTypeIsLocal()) { - try { - if (!Files.exists(sequenceFileDir)) { - Files.createDirectory(sequenceFileDir); - logger.trace("Created directory: [" + sequenceFileDir.toString() + "]"); - } - - if (!Files.exists(sequenceFileDirWithRevision)) { - Files.createDirectory(sequenceFileDirWithRevision); - logger.trace("Created directory: [" + sequenceFileDirWithRevision.toString() + "]"); - } - } catch (IOException e) { - logger.error("Unable to create new directory", e); - throw new StorageException("Failed to create new directory.", e); - } - } - iridaFileStorageService.writeFile(source, target); + iridaFileStorageService.writeFile(source, target, sequenceFileDir, sequenceFileDirWithRevision); ReflectionUtils.setField(field, objectToWrite, target); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index e7226f5ff3a..d10a4daa3f3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -53,7 +53,7 @@ public Long getFileSize(Path file) { * {@inheritDoc} */ @Override - public void writeFile(Path source, Path target) { + public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { // Implement AWS code to upload file to bucket } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 5d6671547c7..7ba2efbf615 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -87,7 +87,7 @@ public Long getFileSize(Path file) { * {@inheritDoc} */ @Override - public void writeFile(Path source, Path target) { + public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(target.toAbsolutePath() .toString() diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index 990d4c129a2..ccaceb20b8e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -59,7 +59,22 @@ public Long getFileSize(Path file) { * {@inheritDoc} */ @Override - public void writeFile(Path source, Path target) { + public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { + try { + if (!Files.exists(sequenceFileDir)) { + Files.createDirectory(sequenceFileDir); + logger.trace("Created directory: [" + sequenceFileDir.toString() + "]"); + } + + if (!Files.exists(sequenceFileDirWithRevision)) { + Files.createDirectory(sequenceFileDirWithRevision); + logger.trace("Created directory: [" + sequenceFileDirWithRevision.toString() + "]"); + } + } catch (IOException e) { + logger.error("Unable to create new directory", e); + throw new StorageException("Failed to create new directory.", e); + } + try { Files.move(source, target); logger.trace("Moved file " + source + " to " + target); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 634f3bb1396..00bf47ca78a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -32,7 +32,7 @@ public interface IridaFileStorageService { * @param source The {@link Path} to the file * @param target The {@link Path} to where file should be moved */ - public void writeFile(Path source, Path target); + public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision); /** * Returns if the storage type is local or not @@ -96,5 +96,4 @@ public interface IridaFileStorageService { * */ public boolean isGzipped(Path file) throws IOException; - } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java index c814d41a374..e8faf32e111 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java @@ -10,6 +10,7 @@ import java.util.Locale; import java.util.Map; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.web.projects.ProjectControllerUtils; import ca.corefacility.bioinformatics.irida.ria.web.projects.ProjectsController; import org.slf4j.Logger; @@ -43,14 +44,16 @@ public class ProjectReferenceFileController { private final ReferenceFileService referenceFileService; private final ProjectControllerUtils projectControllerUtils; private final MessageSource messageSource; + private IridaFileStorageService iridaFileStorageService; @Autowired public ProjectReferenceFileController(ProjectService projectService, ReferenceFileService referenceFileService, - ProjectControllerUtils projectControllerUtils, MessageSource messageSource) { + ProjectControllerUtils projectControllerUtils, MessageSource messageSource, IridaFileStorageService iridaFileStorageService) { this.projectService = projectService; this.referenceFileService = referenceFileService; this.projectControllerUtils = projectControllerUtils; this.messageSource = messageSource; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -93,12 +96,7 @@ public String getProjectReferenceFilesPage(final Model model, final Principal pr map.put("label", file.getLabel()); map.put("createdDate", file.getCreatedDate()); Path path = file.getFile(); - try { - map.put("size", Files.size(path)); - } catch (IOException e) { - logger.error("Cannot find the size of file " + file.getLabel()); - map.put("size", messageSource.getMessage("projects.reference-file.not-found", new Object[] {}, locale)); - } + map.put("size", iridaFileStorageService.getFileSize(path)); files.add(map); } return ImmutableMap.of("files", files); diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml similarity index 56% rename from src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml rename to src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml index 6b73f0bd48c..16ff7998fff 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-sequence-file-type.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml @@ -3,19 +3,17 @@ - - + http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd"> + - - + + - + - - \ No newline at end of file + diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml index f3b59cba36b..a425dd5005a 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml @@ -8,6 +8,7 @@ relativeToChangelogFile="true"/> - + From c8672d656aba541db4571a30762c2c670de2ab59 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Sat, 11 Apr 2020 17:24:22 -0500 Subject: [PATCH 018/655] Removed iridafilestoragefactory and moved sequencefile creation code into iridafilestorageservice --- .../services/IridaApiServicesConfig.java | 6 -- .../SequencingObjectConcatenatorFactory.java | 16 ++--- .../impl/SequenceFilePairConcatenator.java | 12 ++-- .../SingleEndSequenceFileConcatenator.java | 10 ++-- .../IridaFileStorageAwsServiceImpl.java | 34 ++++++++++- .../IridaFileStorageAzureServiceImpl.java | 31 ++++++++++ .../IridaFileStorageLocalServiceImpl.java | 27 +++++++++ .../filesystem/IridaFileStorageService.java | 23 +++++++ .../web/samples/SamplesAjaxController.java | 10 ++-- .../impl/IridaFileStorageFactoryImpl.java | 60 ------------------- .../impl/SequencingObjectServiceImpl.java | 9 +-- .../RESTSampleSequenceFilesController.java | 10 ++-- ...quencingObjectConcatenatorFactoryTest.java | 16 ++--- .../SequenceFilePairConcatenatorTest.java | 6 +- ...SingleEndSequenceFileConcatenatorTest.java | 6 +- .../samples/SamplesAjaxControllerTest.java | 6 +- .../unit/SequencingObjectServiceTest.java | 6 +- .../SampleSequenceFilesControllerTest.java | 28 ++++----- 18 files changed, 180 insertions(+), 136 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 7b81be31f9c..8a268114ab9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -22,7 +22,6 @@ import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionCleanupService; import ca.corefacility.bioinformatics.irida.service.TaxonomyService; import ca.corefacility.bioinformatics.irida.service.impl.InMemoryTaxonomyService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.AnalysisSubmissionCleanupServiceImpl; import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.util.IridaPluginMessageSource; @@ -307,11 +306,6 @@ public IridaFileStorageService iridaFileStorageService() { } } - @Bean(name = "iridaFileStorageFactory") - public IridaFileStorageFactoryImpl iridaFileStorageFactory(IridaFileStorageService iridaFileStorageService) { - return new IridaFileStorageFactoryImpl(iridaFileStorageService); - } - @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, GzipFileProcessor gzipFileProcessor, diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java index cb4c6fa768e..b0e98483f6a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java @@ -9,7 +9,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** * Factory class for returning an instance of @@ -22,17 +22,17 @@ public class SequencingObjectConcatenatorFactory { * * @param type the class to get a concatenator for * @param The type this concatenator should act on - * @param iridaFileStorageFactory The file storage factory component + * @param iridaFileStorageService The file storage component * @return the new {@link SequencingObjectConcatenator} */ @SuppressWarnings("unchecked") - public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageFactoryImpl iridaFileStorageFactory) { + public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageService iridaFileStorageService) { // return the concatenator for the class if (type.equals(SingleEndSequenceFile.class)) { - return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageFactory); + return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageService); } else if (type.equals(SequenceFilePair.class)) { - return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageFactory); + return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageService); } else { throw new IllegalArgumentException("No concatenator exists for type " + type); } @@ -43,11 +43,11 @@ public static SequencingObjectConcatenator getCo * {@link SequencingObject}s * * @param objects the {@link SequencingObject}s to get the concatenator for - * @param iridaFileStorageFactory The file storage factory component + * @param iridaFileStorageService The file storage component * @return the new {@link SequencingObjectConcatenator} */ public static SequencingObjectConcatenator getConcatenator( - Collection objects, IridaFileStorageFactoryImpl iridaFileStorageFactory) { + Collection objects, IridaFileStorageService iridaFileStorageService) { // get all the classes for the objects Set types = objects.stream().map(Object::getClass).collect(Collectors.toSet()); @@ -61,6 +61,6 @@ public static SequencingObjectConcatenator getConcat @SuppressWarnings("unchecked") Class type = (Class) types.iterator().next(); - return getConcatenator(type, iridaFileStorageFactory); + return getConcatenator(type, iridaFileStorageService); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index c5e4306064e..a526cd767fb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import java.io.IOException; import java.nio.file.Files; @@ -21,11 +21,11 @@ @Component public class SequenceFilePairConcatenator extends SequencingObjectConcatenator { - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Autowired - public SequenceFilePairConcatenator(IridaFileStorageFactoryImpl iridaFileStorageFactory) { - this.iridaFileStorageFactory = iridaFileStorageFactory; + public SequenceFilePairConcatenator(IridaFileStorageService iridaFileStorageService) { + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -70,8 +70,8 @@ public SequenceFilePair concatenateFiles(List toConc } // create new SequenceFiles - SequenceFile forward = iridaFileStorageFactory.createSequenceFile(forwardFile); - SequenceFile reverse = iridaFileStorageFactory.createSequenceFile(reverseFile); + SequenceFile forward = iridaFileStorageService.createSequenceFile(forwardFile); + SequenceFile reverse = iridaFileStorageService.createSequenceFile(reverseFile); // create the new pair SequenceFilePair sequenceFilePair = new SequenceFilePair(forward, reverse); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index ceeeb462521..7b285b7495f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import java.io.IOException; import java.nio.file.Files; @@ -21,11 +21,11 @@ @Component public class SingleEndSequenceFileConcatenator extends SequencingObjectConcatenator { - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Autowired - public SingleEndSequenceFileConcatenator(IridaFileStorageFactoryImpl iridaFileStorageFactory) { - this.iridaFileStorageFactory = iridaFileStorageFactory; + public SingleEndSequenceFileConcatenator(IridaFileStorageService iridaFileStorageService) { + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -62,7 +62,7 @@ public SingleEndSequenceFile concatenateFiles(List t } // create the new sequencefile and object - SequenceFile forward = iridaFileStorageFactory.createSequenceFile(tempFile); + SequenceFile forward = iridaFileStorageService.createSequenceFile(tempFile); SingleEndSequenceFile seqObject = new SingleEndSequenceFile(forward); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index d10a4daa3f3..5ddb1618e3c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -12,6 +12,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + /** * Component implementation of file utitlities for aws storage */ @@ -19,9 +22,6 @@ public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); - //Aws Specific Variables - - @Autowired public IridaFileStorageAwsServiceImpl(){ @@ -78,6 +78,9 @@ public void downloadFile() { public void downloadFiles() { } + /** + * {@inheritDoc} + */ @Override public boolean storageTypeIsLocal(){ return false; @@ -92,11 +95,17 @@ public String getFileName(Path file) { return fileName; } + /** + * {@inheritDoc} + */ @Override public boolean fileExists(Path file) { return false; } + /** + * {@inheritDoc} + */ @Override public InputStream getFileInputStream(Path file) { InputStream inputstream = null; @@ -109,6 +118,9 @@ public InputStream getFileInputStream(Path file) { return inputstream; } + /** + * {@inheritDoc} + */ @Override public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { @@ -118,4 +130,20 @@ public boolean isGzipped(Path file) throws IOException { && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createEmptySequenceFile() { + return new CloudSequenceFile(); + } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createSequenceFile(Path file) { + return new CloudSequenceFile(file); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 7ba2efbf615..d4c3a387944 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -12,6 +12,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; @@ -119,6 +122,9 @@ public void downloadFile() { public void downloadFiles() { } + /** + * {@inheritDoc} + */ @Override public boolean storageTypeIsLocal(){ return false; @@ -145,6 +151,9 @@ public String getFileName(Path file) { return fileName; } + /** + * {@inheritDoc} + */ @Override public boolean fileExists(Path file) { blobClient = containerClient.getBlobClient(file.toAbsolutePath() @@ -156,6 +165,9 @@ public boolean fileExists(Path file) { return false; } + /** + * {@inheritDoc} + */ @Override public InputStream getFileInputStream(Path file) { blobClient = containerClient.getBlobClient(file.toAbsolutePath() @@ -164,6 +176,9 @@ public InputStream getFileInputStream(Path file) { return blobClient.openInputStream(); } + /** + * {@inheritDoc} + */ @Override public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { @@ -173,4 +188,20 @@ public boolean isGzipped(Path file) throws IOException { && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createEmptySequenceFile() { + return new CloudSequenceFile(); + } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createSequenceFile(Path file) { + return new CloudSequenceFile(file); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index ccaceb20b8e..da70c158447 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -15,6 +15,8 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; /** @@ -121,11 +123,17 @@ public String getFileName(Path file) { return fileName; } + /** + * {@inheritDoc} + */ @Override public boolean fileExists(Path file) { return Files.exists(file); } + /** + * {@inheritDoc} + */ @Override public InputStream getFileInputStream(Path file) { try { @@ -135,6 +143,9 @@ public InputStream getFileInputStream(Path file) { } } + /** + * {@inheritDoc} + */ @Override public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { @@ -144,4 +155,20 @@ public boolean isGzipped(Path file) throws IOException { && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createEmptySequenceFile() { + return new LocalSequenceFile(); + } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createSequenceFile(Path file) { + return new LocalSequenceFile(file); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 00bf47ca78a..58107716339 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -5,6 +5,8 @@ import java.io.InputStream; import java.nio.file.Path; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + /** * Interface describing methods for performing storage actions */ @@ -96,4 +98,25 @@ public interface IridaFileStorageService { * */ public boolean isGzipped(Path file) throws IOException; + + /** + * Creates an empty SequenceFile depending on + * storage type + * + * @return Local or Cloud {@link SequenceFile} object + * @throws IOException if file can't be read + * + */ + public SequenceFile createEmptySequenceFile(); + + /** + * Creates a SequenceFile depending on + * storage type + * + * @param file The path to the file + * @return Local or Cloud {@link SequenceFile} object + * @throws IOException if file can't be read + * + */ + public SequenceFile createSequenceFile(Path file); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index b69161bab6d..cdc622cb33b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -21,9 +21,9 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; /** @@ -36,16 +36,16 @@ public class SamplesAjaxController { private final SequencingObjectService sequencingObjectService; private final GenomeAssemblyService genomeAssemblyService; private final MessageSource messageSource; - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Autowired public SamplesAjaxController(SampleService sampleService, SequencingObjectService sequencingObjectService, - GenomeAssemblyService genomeAssemblyService, MessageSource messageSource, IridaFileStorageFactoryImpl iridaFileStorageFactory) { + GenomeAssemblyService genomeAssemblyService, MessageSource messageSource, IridaFileStorageService iridaFileStorageService) { this.sampleService = sampleService; this.sequencingObjectService = sequencingObjectService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; - this.iridaFileStorageFactory = iridaFileStorageFactory; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -162,6 +162,6 @@ private SequenceFile createSequenceFile(MultipartFile file) throws IOException { Path temp = Files.createTempDirectory(null); Path target = temp.resolve(file.getOriginalFilename()); file.transferTo(target.toFile()); - return iridaFileStorageFactory.createSequenceFile(target); + return iridaFileStorageService.createSequenceFile(target); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java deleted file mode 100644 index 721b7fa768a..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/IridaFileStorageFactoryImpl.java +++ /dev/null @@ -1,60 +0,0 @@ -package ca.corefacility.bioinformatics.irida.service.impl; - -import java.nio.file.Path; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; - -/** - * Service implementation for creating SequenceFiles {@link LocalSequenceFile} and {@link CloudSequenceFile} - */ -@Service -public class IridaFileStorageFactoryImpl { - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageFactoryImpl.class); - private IridaFileStorageService iridaFileStorageService; - - @Autowired - public IridaFileStorageFactoryImpl(IridaFileStorageService iridaFileStorageService) { - this.iridaFileStorageService = iridaFileStorageService; - } - - /** - * Creates a {@link LocalSequenceFile} or {@link CloudSequenceFile} depending - * on the storage type. - * - * @param file The path to the file for which to create an object for - * @return a {@link LocalSequenceFile} or {@link CloudSequenceFile} as - * a {@link SequenceFile} - */ - public SequenceFile createSequenceFile(Path file) { - if (iridaFileStorageService.storageTypeIsLocal()) { - return new LocalSequenceFile(file); - } else { - return new CloudSequenceFile(file); - } - } - - /** - * Creates an empty {@link LocalSequenceFile} or {@link CloudSequenceFile} depending - * on the storage type. - * - * @return an empty {@link LocalSequenceFile} or {@link CloudSequenceFile} object as - * a {@link SequenceFile} - */ - public SequenceFile createEmptySequenceFile() { - if (iridaFileStorageService.storageTypeIsLocal()) { - return new LocalSequenceFile(); - - } else { - return new CloudSequenceFile(); - } - } - -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index 29a22bf5421..a118b7f6483 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -12,6 +12,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenatorFactory; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -44,19 +45,19 @@ public class SequencingObjectServiceImpl extends CRUDServiceImpl to throws ConcatenateException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory - .getConcatenator(toJoin, iridaFileStorageFactory); + .getConcatenator(toJoin, iridaFileStorageService); SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 0c8270cfe9f..08763e8cace 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -36,8 +36,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.AnalysisService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -135,19 +135,19 @@ public class RESTSampleSequenceFilesController { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; protected RESTSampleSequenceFilesController() { } @Autowired public RESTSampleSequenceFilesController(SampleService sampleService, SequencingRunService miseqRunService, - SequencingObjectService sequencingObjectService, AnalysisService analysisService, IridaFileStorageFactoryImpl iridaFileStorageFactory) { + SequencingObjectService sequencingObjectService, AnalysisService analysisService, IridaFileStorageService iridaFileStorageService) { this.sampleService = sampleService; this.miseqRunService = miseqRunService; this.sequencingObjectService = sequencingObjectService; this.analysisService = analysisService; - this.iridaFileStorageFactory = iridaFileStorageFactory; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -417,7 +417,7 @@ public ModelMap addNewSequenceFileToSample(@PathVariable Long sampleId, @Request logger.trace("Read miseq run " + miseqRunId); } } else { - sf = iridaFileStorageFactory.createEmptySequenceFile(); + sf = iridaFileStorageService.createEmptySequenceFile(); } sf.setFile(target); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index 6aa4d4da3fd..560048aca52 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -6,7 +6,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Sets; import org.junit.Test; @@ -30,44 +30,44 @@ public class SequencingObjectConcatenatorFactoryTest { @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class, iridaFileStorageFactory); + SingleEndSequenceFile.class, iridaFileStorageService); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class, iridaFileStorageFactory); + SequenceFilePair.class, iridaFileStorageService); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageFactory); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageService); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageFactory); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageFactory); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageFactory); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 5e05f43b35c..b82eaad5f0c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Lists; import org.junit.Before; @@ -34,11 +34,11 @@ public class SequenceFilePairConcatenatorTest { private SequenceFilePairConcatenator concat; @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { - concat = new SequenceFilePairConcatenator(iridaFileStorageFactory); + concat = new SequenceFilePairConcatenator(iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 52281b206f1..8e3fef09d8b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Lists; import org.junit.Before; @@ -34,11 +34,11 @@ public class SingleEndSequenceFileConcatenatorTest { SingleEndSequenceFileConcatenator concat; @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { - concat = new SingleEndSequenceFileConcatenator(iridaFileStorageFactory); + concat = new SingleEndSequenceFileConcatenator(iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 043c05cf09a..bbbbf539c36 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -25,11 +25,11 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.google.common.collect.ImmutableList; @@ -47,7 +47,7 @@ public class SamplesAjaxControllerTest { private GenomeAssemblyService genomeAssemblyService; @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; /* TEST DATA @@ -71,7 +71,7 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource, iridaFileStorageFactory); + controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource, iridaFileStorageService); // Set up mocks when(sampleService.read(SAMPLE.getId())).thenReturn(SAMPLE); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java index 3203a7dad10..f72072517b2 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java @@ -7,12 +7,12 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.impl.SequencingObjectServiceImpl; import ca.corefacility.bioinformatics.irida.web.controller.test.unit.TestDataFactory; import org.junit.Before; @@ -42,7 +42,7 @@ public class SequencingObjectServiceTest { Validator validator; @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { @@ -53,7 +53,7 @@ public void setUp() { concatenationRepository = mock(SequenceConcatenationRepository.class); service = new SequencingObjectServiceImpl(repository, sequenceFileRepository, ssoRepository, - concatenationRepository, validator, iridaFileStorageFactory); + concatenationRepository, validator, iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index f6703a51bdb..49f163f7a09 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -46,10 +46,10 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; -import ca.corefacility.bioinformatics.irida.service.impl.IridaFileStorageFactoryImpl; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResourceCollection; import ca.corefacility.bioinformatics.irida.web.assembler.resource.RootResource; @@ -73,7 +73,7 @@ public class SampleSequenceFilesControllerTest { private SequencingRun sequencingRun; @Autowired - private IridaFileStorageFactoryImpl iridaFileStorageFactory; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { @@ -83,7 +83,7 @@ public void setUp() { analysisService = mock(AnalysisService.class); sequencingRun = mock(SequencingRun.class); - controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService, iridaFileStorageFactory); + controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService, iridaFileStorageService); } @Test @@ -245,7 +245,7 @@ public void testAddNewSequenceFileToSample() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(6L); @@ -297,7 +297,7 @@ public void testAddNewSequenceFileToSampleCompletedRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(8L); @@ -323,7 +323,7 @@ public void testAddNewSequenceFileToSampleErrorRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(8L); Path f = Files.createTempFile(null, null); @@ -360,8 +360,8 @@ public void testAddNewSequenceFilePairToSample() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); @@ -437,8 +437,8 @@ public void testAddNewSequenceFilePairToSampleMismatchedRunIDs() throws IOExcept SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); @@ -466,8 +466,8 @@ public void testAddNewSequenceFilePairToSampleCompletedRun() throws IOException SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); @@ -499,8 +499,8 @@ public void testAddNewSequenceFilePairToSampleErrorRun() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageFactory.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageFactory.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); From 8fa814cbe476213a6f6faa61dc43a61b5da9cbcd Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Apr 2020 13:58:50 -0500 Subject: [PATCH 019/655] Moved appendToFile and getFileExtension for the sequencefileconcatenator to the implementations for the iridafilestorageservice. Refactored code in azure iridafilestorage service --- .../SequencingObjectConcatenator.java | 64 ----------- .../impl/SequenceFilePairConcatenator.java | 6 +- .../SingleEndSequenceFileConcatenator.java | 4 +- .../IridaFileStorageAwsServiceImpl.java | 37 ++++++- .../IridaFileStorageAzureServiceImpl.java | 102 +++++++++++++++--- .../IridaFileStorageLocalServiceImpl.java | 62 +++++++++++ .../filesystem/IridaFileStorageService.java | 25 +++++ 7 files changed, 213 insertions(+), 87 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index ba12b231c98..d8b11a718e9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -19,8 +19,6 @@ * @param the {@link SequencingObject} class to concatenate */ public abstract class SequencingObjectConcatenator { - //Valid extensions to try to concatenate with this tool - private static final List VALID_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** * Concatenate a set of {@link SequencingObject}s of a given type @@ -33,66 +31,4 @@ public abstract class SequencingObjectConcatenator toConcatenate, String filename) throws ConcatenateException; - /** - * Append a {@link SequenceFile} to a {@link Path} on the filesystem - * - * @param target the {@link Path} to append to - * @param file the {@link SequenceFile} to append to the path - * @throws ConcatenateException if there is an error appending the file - */ - protected void appendToFile(Path target, SequenceFile file) throws ConcatenateException { - - try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, - StandardOpenOption.WRITE)) { - try (FileChannel in = FileChannel.open(file.getFile(), StandardOpenOption.READ)) { - for (long p = 0, l = in.size(); p < l; ) { - p += in.transferTo(p, l - p, out); - } - } catch (IOException e) { - throw new ConcatenateException("Could not open input file for reading", e); - } - - } catch (IOException e) { - throw new ConcatenateException("Could not open target file for writing", e); - } - } - - /** - * Get the extension of the files to concatenate - * - * @param toConcatenate The list of {@link SequencingObject} to concatenate - * @return The common extension of the files - * @throws ConcatenateException if the files have different or invalid extensions - */ - protected String getFileExtension(List toConcatenate) throws ConcatenateException { - String selectedExtension = null; - for (SequencingObject object : toConcatenate) { - - for (SequenceFile file : object.getFiles()) { - String fileName = file.getFile() - .toFile() - .getName(); - - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() - .filter(e -> fileName.endsWith(e)) - .findFirst(); - - if (!currentExtensionOpt.isPresent()) { - throw new ConcatenateException("File extension is not valid " + fileName); - } - - String currentExtension = currentExtensionOpt.get(); - - if (selectedExtension == null) { - selectedExtension = currentExtensionOpt.get(); - } else if (selectedExtension != currentExtensionOpt.get()) { - throw new ConcatenateException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " - + selectedExtension); - } - } - } - - return selectedExtension; - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index a526cd767fb..094be469685 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -35,7 +35,7 @@ public SequenceFilePairConcatenator(IridaFileStorageService iridaFileStorageServ public SequenceFilePair concatenateFiles(List toConcatenate, String filename) throws ConcatenateException { - String extension = getFileExtension(toConcatenate); + String extension = iridaFileStorageService.getFileExtension(toConcatenate); // create the filenames with F/R for the forward and reverse files String forwardName = filename + "_R1." + extension; @@ -65,8 +65,8 @@ public SequenceFilePair concatenateFiles(List toConc SequenceFile forwardSequenceFile = pair.getForwardSequenceFile(); SequenceFile reverseSequenceFile = pair.getReverseSequenceFile(); - appendToFile(forwardFile, forwardSequenceFile); - appendToFile(reverseFile, reverseSequenceFile); + iridaFileStorageService.appendToFile(forwardFile, forwardSequenceFile); + iridaFileStorageService.appendToFile(reverseFile, reverseSequenceFile); } // create new SequenceFiles diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 7b285b7495f..0aee792cf16 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -36,7 +36,7 @@ public SingleEndSequenceFile concatenateFiles(List t throws ConcatenateException { Path tempFile; - String extension = getFileExtension(toConcatenate); + String extension = iridaFileStorageService.getFileExtension(toConcatenate); // create the filename with extension filename = filename + "." + extension; @@ -58,7 +58,7 @@ public SingleEndSequenceFile concatenateFiles(List t SequenceFile forwardSequenceFile = single.getSequenceFile(); - appendToFile(tempFile, forwardSequenceFile); + iridaFileStorageService.appendToFile(tempFile, forwardSequenceFile); } // create the new sequencefile and object diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 5ddb1618e3c..8cc4016e9fa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -4,7 +4,11 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.FileChannel; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Optional; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; @@ -12,8 +16,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; + +import com.google.common.collect.Lists; /** * Component implementation of file utitlities for aws storage @@ -24,7 +32,6 @@ public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ @Autowired public IridaFileStorageAwsServiceImpl(){ - } /** @@ -146,4 +153,32 @@ public SequenceFile createEmptySequenceFile() { public SequenceFile createSequenceFile(Path file) { return new CloudSequenceFile(file); } + + /** + * Removes the leading "/" from the absolute path + * returns the rest of the path. + * + * @param file + * @return + */ + private String getAwsFileAbsolutePath(Path file) { + String absolutePath = file.toAbsolutePath().toString(); + return absolutePath; + } + + /** + * {@inheritDoc} + */ + @Override + public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + } + + /** + * {@inheritDoc} + */ + @Override + public String getFileExtension(List toConcatenate) throws ConcatenateException { + String selectedExtension = null; + return selectedExtension; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index d4c3a387944..7e5baa67287 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -1,10 +1,15 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.FileChannel; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Date; +import java.util.List; +import java.util.Optional; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; @@ -12,14 +17,17 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobStorageException; +import com.google.common.collect.Lists; /** * Component implementation of file utitlities for azure storage @@ -47,9 +55,7 @@ public File getTemporaryFile(Path file) { File fileToProcess = null; // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString() - .substring(1)); + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { // Create a file that will be unique in the /tmp/ folder. We append the current date/time @@ -77,8 +83,7 @@ public Long getFileSize(Path file) { Long fileSize = 0L; try { // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString().substring(1)); + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = blobClient.getProperties().getBlobSize(); } catch (BlobStorageException e) { logger.debug("Couldn't calculate size as the file was not found [" + e + "]"); @@ -92,9 +97,7 @@ public Long getFileSize(Path file) { @Override public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(target.toAbsolutePath() - .toString() - .substring(1)); + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); logger.debug("Uploading file to azure: [" + target.getFileName() + "]"); blobClient.uploadFromFile(source.toString(), false); @@ -135,9 +138,7 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString() - .substring(1)); + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { // Since the file system is virtual the full file path is the file name. // We split it on "/" and get the last token which is the actual file name. @@ -156,9 +157,7 @@ public String getFileName(Path file) { */ @Override public boolean fileExists(Path file) { - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString() - .substring(1)); + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); if(blobClient.getProperties().getBlobSize() > 0) { return true; } @@ -170,9 +169,7 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { - blobClient = containerClient.getBlobClient(file.toAbsolutePath() - .toString() - .substring(1)); + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); return blobClient.openInputStream(); } @@ -204,4 +201,75 @@ public SequenceFile createEmptySequenceFile() { public SequenceFile createSequenceFile(Path file) { return new CloudSequenceFile(file); } + + /** + * Removes the leading "/" from the absolute path + * returns the rest of the path. + * + * @param file + * @return + */ + private String getAzureFileAbsolutePath(Path file) { + String absolutePath = file.toAbsolutePath().toString(); + if(absolutePath.charAt(0) == '/') { + absolutePath = file.toAbsolutePath() + .toString() + .substring(1); + } + return absolutePath; + } + + /** + * {@inheritDoc} + */ + @Override + public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, + StandardOpenOption.WRITE)) { + try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { + for (long p = 0, l = in.size(); p < l; ) { + p += in.transferTo(p, l - p, out); + } + } catch (IOException e) { + throw new ConcatenateException("Could not open input file for reading", e); + } + + } catch (IOException e) { + throw new ConcatenateException("Could not open target file for writing", e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public String getFileExtension(List toConcatenate) throws ConcatenateException { + String selectedExtension = null; + for (SequencingObject object : toConcatenate) { + + for (SequenceFile file : object.getFiles()) { + String fileName = getFileName(file.getFile()); + + Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + .filter(e -> fileName.endsWith(e)) + .findFirst(); + + if (!currentExtensionOpt.isPresent()) { + throw new ConcatenateException("File extension is not valid " + fileName); + } + + String currentExtension = currentExtensionOpt.get(); + + if (selectedExtension == null) { + selectedExtension = currentExtensionOpt.get(); + } else if (selectedExtension != currentExtensionOpt.get()) { + throw new ConcatenateException( + "Extensions of files to concatenate do not match " + currentExtension + " vs " + + selectedExtension); + } + } + } + + return selectedExtension; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index da70c158447..006fe821f53 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -3,10 +3,13 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Optional; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; @@ -14,11 +17,15 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import com.google.common.collect.Lists; + /** * Component implementation of file utitlities for local storage */ @@ -171,4 +178,59 @@ public SequenceFile createEmptySequenceFile() { public SequenceFile createSequenceFile(Path file) { return new LocalSequenceFile(file); } + + /** + * {@inheritDoc} + */ + public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + + try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, + StandardOpenOption.WRITE)) { + try (FileChannel in = FileChannel.open(file.getFile(), StandardOpenOption.READ)) { + for (long p = 0, l = in.size(); p < l; ) { + p += in.transferTo(p, l - p, out); + } + } catch (IOException e) { + throw new ConcatenateException("Could not open input file for reading", e); + } + + } catch (IOException e) { + throw new ConcatenateException("Could not open target file for writing", e); + } + } + + /** + * {@inheritDoc} + */ + public String getFileExtension(List toConcatenate) throws ConcatenateException { + String selectedExtension = null; + for (SequencingObject object : toConcatenate) { + + for (SequenceFile file : object.getFiles()) { + String fileName = file.getFile() + .toFile() + .getName(); + + Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + .filter(e -> fileName.endsWith(e)) + .findFirst(); + + if (!currentExtensionOpt.isPresent()) { + throw new ConcatenateException("File extension is not valid " + fileName); + } + + String currentExtension = currentExtensionOpt.get(); + + if (selectedExtension == null) { + selectedExtension = currentExtensionOpt.get(); + } else if (selectedExtension != currentExtensionOpt.get()) { + throw new ConcatenateException( + "Extensions of files to concatenate do not match " + currentExtension + " vs " + + selectedExtension); + } + } + } + + return selectedExtension; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 58107716339..d0a50490c2d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -4,14 +4,21 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.List; +import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; + +import com.google.common.collect.Lists; /** * Interface describing methods for performing storage actions */ public interface IridaFileStorageService { + //Valid extensions to try to concatenate with this tool + public static final List VALID_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** * Get a temporarry file from storage * @@ -119,4 +126,22 @@ public interface IridaFileStorageService { * */ public SequenceFile createSequenceFile(Path file); + + /** + * Append a {@link SequenceFile} to a {@link Path} on the filesystem + * + * @param target the {@link Path} to append to + * @param file the {@link SequenceFile} to append to the path + * @throws ConcatenateException if there is an error appending the file + */ + public void appendToFile(Path target, SequenceFile file) throws ConcatenateException; + + /** + * Get the extension of the files to concatenate + * + * @param toConcatenate The list of {@link SequencingObject} to concatenate + * @return The common extension of the files + * @throws ConcatenateException if the files have different or invalid extensions + */ + public String getFileExtension(List toConcatenate) throws ConcatenateException; } From 8b5c5b0bae0b5fdf7413803d2c2fc93218c0cba4 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Apr 2020 16:03:27 -0500 Subject: [PATCH 020/655] Added a listener for GenomeAssembly to give the model access to iridaFileStorageService after loading the model. --- .../irida/model/assembly/GenomeAssembly.java | 28 ++++++------------ .../model/sequenceFile/SequenceFile.java | 4 +-- .../listeners/GenomeAssemblyListener.java | 29 +++++++++++++++++++ ...istener.java => SequenceFileListener.java} | 5 +--- 4 files changed, 41 insertions(+), 25 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java rename src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/{IridaFileStorageServiceListener.java => SequenceFileListener.java} (84%) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index c824dbba4eb..66608db4591 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.Date; import java.util.List; @@ -21,6 +20,8 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; import ca.corefacility.bioinformatics.irida.model.joins.impl.SampleGenomeAssemblyJoin; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.GenomeAssemblyListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Lists; @@ -31,11 +32,13 @@ @Entity @Table(name = "genome_assembly") @Inheritance(strategy = InheritanceType.JOINED) +@EntityListeners({ GenomeAssemblyListener.class }) @Audited public abstract class GenomeAssembly extends IridaResourceSupport implements IridaThing, IridaSequenceFile, VersionedFileFields { private static final Logger logger = LoggerFactory.getLogger(GenomeAssembly.class); + private static IridaFileStorageService iridaFileStorageService; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -77,17 +80,6 @@ public Date getCreatedDate() { return createdDate; } - /** - * Get the size of the genome assembly files - * - * @return file size - * @throws IOException if the file cannot be read - */ - @JsonIgnore - public long getFileSizeBytes() throws IOException { - return Files.size(getFile()); - } - /** * Add a sample to this assembly * @@ -107,13 +99,7 @@ public void addSampleGenomeAssemblyJoin(SampleGenomeAssemblyJoin join) { @JsonIgnore public String getFileSize() { String size = "N/A"; - try { - size = IridaSequenceFile.humanReadableByteCount(getFileSizeBytes(), true); - } catch (NoSuchFileException e) { - logger.error("Could not find file " + getFile()); - } catch (IOException e) { - logger.error("Could not calculate file size: ", e); - } + size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageService.getFileSize(getFile()), true); return size; } @@ -144,4 +130,8 @@ public boolean equals(Object obj) { GenomeAssembly other = (GenomeAssembly) obj; return Objects.equals(this.id, other.id) && Objects.equals(this.createdDate, other.createdDate); } + + public void setIridaFileStorageService(IridaFileStorageService iridaFileStorageService) { + this.iridaFileStorageService = iridaFileStorageService; + } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 1cfda2e7b0f..f4fc7b6439c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -26,7 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.remote.RemoteSynchronizable; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageServiceListener; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.SequenceFileListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; @@ -43,7 +43,7 @@ @Inheritance(strategy=InheritanceType.SINGLE_TABLE) @Table(name = "sequence_file") @Audited -@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageServiceListener.class }) +@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, SequenceFileListener.class }) public abstract class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, VersionedFileFields, IridaSequenceFile, RemoteSynchronizable { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java new file mode 100644 index 00000000000..f6741476f5a --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java @@ -0,0 +1,29 @@ +package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; + +import javax.persistence.PostLoad; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; + +import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + +@Component +public class GenomeAssemblyListener { + + @Autowired + private IridaFileStorageService iridaFileStorageService; + + /** + * After the GenomeAssembly entity is loaded this method will provide + * the entity access to the iridaFileStorageService + * + * @param genomeAssembly The entity to provide the iridaFileStorageService to + */ + @PostLoad + public void afterAssemblyFileLoad(GenomeAssembly genomeAssembly) { + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); + genomeAssembly.setIridaFileStorageService(iridaFileStorageService); + } +} \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageServiceListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java similarity index 84% rename from src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageServiceListener.java rename to src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java index e9e3e733cc5..f95d3185f53 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageServiceListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java @@ -2,8 +2,6 @@ import javax.persistence.PostLoad; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; @@ -15,8 +13,7 @@ * Component implementation to run on an entity after it is has been accessed from the db. */ @Component -public class IridaFileStorageServiceListener { - private final Logger logger = LoggerFactory.getLogger(IridaFileStorageServiceListener.class); +public class SequenceFileListener { @Autowired private IridaFileStorageService iridaFileStorageService; From 3569c6bffc239051ba0b600141464aff94fcdd6e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Apr 2020 18:19:28 -0500 Subject: [PATCH 021/655] Changed 'private Path' to 'protected Path' so we don't need to use .getSuperClass in the writeFilesToDisk method. Changed debug statements in iridafilestorage azure service to trace. --- .../irida/model/assembly/UploadedAssembly.java | 2 +- .../irida/model/project/ReferenceFile.java | 2 +- .../irida/model/sequenceFile/SequenceFile.java | 2 +- .../model/workflow/analysis/AnalysisOutputFile.java | 2 +- .../FilesystemSupplementedRepositoryImpl.java | 12 +++--------- .../filesystem/IridaFileStorageAwsServiceImpl.java | 6 +----- .../filesystem/IridaFileStorageAzureServiceImpl.java | 11 +++++------ .../filesystem/IridaFileStorageLocalServiceImpl.java | 2 -- 8 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java index 94d39976efd..23adfa8d04e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java @@ -24,7 +24,7 @@ public class UploadedAssembly extends GenomeAssembly implements VersionedFileFie @NotNull @Column(name = "file_path", unique = true) - private Path file; + protected Path file; @Column(name = "file_revision_number") Long fileRevisionNumber; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index f88a1d57a15..8ca4aee9315 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -44,7 +44,7 @@ public class ReferenceFile implements VersionedFileFields, MutableIridaThi @Column(name = "filePath", unique = true) @NotNull(message = "{reference.file.file.notnull}") - private Path file; + protected Path file; @CreatedDate @NotNull diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index f4fc7b6439c..0f7712d5f12 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -57,7 +57,7 @@ public abstract class SequenceFile extends IridaResourceSupport implements Mutab @NotNull(message = "{sequencefile.file.notnull}") @Column(name = "file_path", unique = true) - private Path file; + protected Path file; @CreatedDate @NotNull diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 0bb0bcac3bb..c7f8013c705 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -46,7 +46,7 @@ public class AnalysisOutputFile extends IridaResourceSupport implements IridaThi @NotNull(message = "{analysis.output.file.file.notnull}") @com.fasterxml.jackson.annotation.JsonIgnore @org.codehaus.jackson.annotate.JsonIgnore - private final Path file; + protected final Path file; @NotNull @Temporal(TemporalType.TIMESTAMP) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index 48bbd1ce3e8..c1181734144 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -1,8 +1,6 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.IOException; import java.lang.reflect.Field; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.HashSet; @@ -18,15 +16,16 @@ import javax.persistence.PostUpdate; import javax.persistence.PreUpdate; +import org.apache.commons.lang3.reflect.FieldUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.ReflectionUtils; -import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; + /** * Custom implementation of a repository that writes the {@link Path} part of an * entity to disk. @@ -220,14 +219,9 @@ private Type writeFilesToDisk(Path baseDirectory, Type objectToWrite) { Predicate pathFilter = f -> f.getType().equals(Path.class); // now find any members that are of type Path and shuffle them around: - Set pathFields = Arrays.stream(objectToWrite.getClass().getSuperclass().getDeclaredFields()).filter(pathFilter) + Set pathFields = Arrays.stream(FieldUtils.getAllFields(objectToWrite.getClass())).filter(pathFilter) .collect(Collectors.toSet()); - if(pathFields.size() == 0) { - pathFields = Arrays.stream(objectToWrite.getClass().getDeclaredFields()).filter(pathFilter) - .collect(Collectors.toSet()); - } - Set fieldsToUpdate = new HashSet<>(); for (Field field : pathFields) { ReflectionUtils.makeAccessible(field); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 8cc4016e9fa..064c8be5e34 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -4,11 +4,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.channels.FileChannel; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.List; -import java.util.Optional; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; @@ -21,14 +18,13 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import com.google.common.collect.Lists; /** * Component implementation of file utitlities for aws storage */ @Component public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsServiceImpl.class); @Autowired public IridaFileStorageAwsServiceImpl(){ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 7e5baa67287..08796c8d46c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -27,7 +27,6 @@ import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobStorageException; -import com.google.common.collect.Lists; /** * Component implementation of file utitlities for azure storage @@ -69,7 +68,7 @@ public File getTemporaryFile(Path file) { blobClient.downloadToFile(filePath); fileToProcess = new File(filePath); } catch (BlobStorageException e) { - logger.debug("Couldn't find file [" + e + "]"); + logger.trace("Couldn't find file [" + e + "]"); } return fileToProcess; @@ -86,7 +85,7 @@ public Long getFileSize(Path file) { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = blobClient.getProperties().getBlobSize(); } catch (BlobStorageException e) { - logger.debug("Couldn't calculate size as the file was not found [" + e + "]"); + logger.trace("Couldn't calculate size as the file was not found [" + e + "]"); } return fileSize; } @@ -99,9 +98,9 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); - logger.debug("Uploading file to azure: [" + target.getFileName() + "]"); + logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); blobClient.uploadFromFile(source.toString(), false); - logger.debug("File uploaded to: [" + blobClient.getBlobUrl() + "]"); + logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); } /** @@ -146,7 +145,7 @@ public String getFileName(Path file) { .split("/"); fileName = blobNameTokens[blobNameTokens.length - 1]; } catch (BlobStorageException e) { - logger.debug("Couldn't find file [" + e + "]"); + logger.trace("Couldn't find file [" + e + "]"); } return fileName; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index 006fe821f53..33209cead37 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -24,8 +24,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; -import com.google.common.collect.Lists; - /** * Component implementation of file utitlities for local storage */ From 781c7b57d31768b153f1b57f8ef7502d6f401f93 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Apr 2020 18:22:22 -0500 Subject: [PATCH 022/655] Updated jackson version to 2.10.3 as azure blob requires >= 2.10.1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8c9d3bc9c56..9d1e527d4fc 100644 --- a/pom.xml +++ b/pom.xml @@ -347,7 +347,7 @@ com.fasterxml.jackson.core jackson-core - 2.10.3 + ${jackson.version} com.fasterxml.jackson.dataformat @@ -1168,7 +1168,7 @@ 1.4.01 3.5.1 3.0 - 2.9.10 + 2.10.3 2.3.4 1.3.1 3.0.0 From 5763a6e2ce36070845a8e09779df4a70e5c10b55 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 14 Apr 2020 15:37:27 -0500 Subject: [PATCH 023/655] Added aws iridafilestorageservice to upload files to a s3 bucket --- pom.xml | 21 +- .../services/IridaApiServicesConfig.java | 15 +- .../IridaFileStorageAwsServiceImpl.java | 180 +++++++++++++++--- 3 files changed, 183 insertions(+), 33 deletions(-) diff --git a/pom.xml b/pom.xml index 9d1e527d4fc..1d3a4a27c5a 100644 --- a/pom.xml +++ b/pom.xml @@ -681,7 +681,7 @@ com.azure azure-storage-blob - 12.0.0 + ${azure-storage-blob.version} jakarta.xml.bind @@ -697,7 +697,7 @@ com.amazonaws aws-java-sdk-s3 - 1.11.376 + ${aws-java-sdk-s3.version} commons-logging @@ -706,6 +706,20 @@ + + + org.apache.httpcomponents + httpclient + ${org.apache.httpcomponents.httpclient.version} + + + commons-logging + commons-logging + + + + + com.monitorjbl @@ -1198,6 +1212,9 @@ 1.3 1.2.0 2.1.0 + 12.0.0 + 1.11.376 + 4.5.12 2.22.1 diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 8a268114ab9..289658aef25 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -137,6 +137,19 @@ public class IridaApiServicesConfig { @Value("${azure.account.connection.string:#{null}}") private String connectionStr; + @Value("${aws.bucket.name:#{null}}") + private String awsBucketName; + + @Value("${aws.bucket.region:#{null}}") + private String awsBucketRegion; + + @Value("${aws.access.key:#{null}}") + private String awsAccessKey; + + @Value("${aws.secret.key:#{null}}") + private String awsSecretKey; + + @Autowired private IridaPluginConfig.IridaPluginList pipelinePlugins; @@ -300,7 +313,7 @@ public IridaFileStorageService iridaFileStorageService() { if(storageType.equalsIgnoreCase("azure")) { return new IridaFileStorageAzureServiceImpl(connectionStr, containerName); } else if (storageType.equalsIgnoreCase("aws")) { - return new IridaFileStorageAwsServiceImpl(); + return new IridaFileStorageAwsServiceImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); } else { return new IridaFileStorageLocalServiceImpl(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 064c8be5e34..bf8d9c59f64 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -1,11 +1,12 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; +import java.nio.channels.FileChannel; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; @@ -18,6 +19,14 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; /** * Component implementation of file utitlities for aws storage @@ -26,8 +35,16 @@ public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsServiceImpl.class); + private String bucketName; + private BasicAWSCredentials awsCreds; + private AmazonS3 s3; + @Autowired - public IridaFileStorageAwsServiceImpl(){ + public IridaFileStorageAwsServiceImpl(String bucketName, String bucketRegion, String accessKey, String secretKey){ + this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); + this.s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion)) + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); + this.bucketName = bucketName; } /** @@ -37,7 +54,39 @@ public IridaFileStorageAwsServiceImpl(){ public File getTemporaryFile(Path file) { File fileToProcess = null; - // Implement AWS code to get file + try { + + String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); + + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. + String [] nameTokens = file.toAbsolutePath().toString().split("/"); + String fileName = nameTokens[nameTokens.length-1]; + + S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); + + fileToProcess = new File(tmpDir+fileName); + + // Stream the file from the bucket into a local file + FileOutputStream fileOutputStream = new FileOutputStream(fileToProcess); + logger.trace("Downloading s3 object to: " + fileToProcess.getAbsolutePath()); + + byte[] read_buf = new byte[1024]; + int read_len = 0; + while ((read_len = s3ObjectInputStream.read(read_buf)) > 0) { + fileOutputStream.write(read_buf, 0, read_len); + } + s3ObjectInputStream.close(); + fileOutputStream.close(); + s3Object.close(); + } catch (AmazonServiceException e) { + logger.error(e.getErrorMessage()); + } catch (FileNotFoundException e) { + logger.error(e.getMessage()); + } catch (IOException e) { + logger.error(e.getMessage()); + } return fileToProcess; } @@ -48,7 +97,16 @@ public File getTemporaryFile(Path file) { @Override public Long getFileSize(Path file) { Long fileSize = 0L; - // Implement AWS code to get file size + try { + S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + fileSize = s3Object.getObjectMetadata() + .getContentLength(); + s3Object.close(); + } catch (AmazonServiceException e) { + logger.error("Unable to get file size from s3 bucket: " + e); + } catch (IOException e) { + logger.error("Unable to close connection to s3object: " + e); + } return fileSize; } @@ -57,7 +115,13 @@ public Long getFileSize(Path file) { */ @Override public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { - // Implement AWS code to upload file to bucket + try { + logger.trace("Uploading file to s3 bucket: [" + target.getFileName() + "]"); + s3.putObject(bucketName, getAwsFileAbsolutePath(target), source.toFile()); + logger.trace("File uploaded to s3 bucket: [" + s3.getUrl(bucketName, target.toAbsolutePath().toString()) + "]"); + } catch (AmazonServiceException e) { + logger.error("Unable to upload file to s3 bucket: " + e); + } } /** @@ -94,7 +158,20 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - // Implement AWS code to get file name + try { + S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. + String[] nameTokens = s3Object.getKey() + .split("/"); + fileName = nameTokens[nameTokens.length - 1]; + s3Object.close(); + } catch (AmazonServiceException e) { + logger.error("Couldn't find file [" + e + "]"); + } catch (IOException e) { + logger.error("Unable to close connection to s3object: " + e); + } + return fileName; } @@ -103,7 +180,7 @@ public String getFileName(Path file) { */ @Override public boolean fileExists(Path file) { - return false; + return s3.doesObjectExist(bucketName, getAwsFileAbsolutePath(file)); } /** @@ -111,14 +188,14 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { - InputStream inputstream = null; + InputStream inputStream = null; try { - inputstream = new FileInputStream(file.toString()); - - } catch(IOException e) { - + S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + inputStream = s3Object.getObjectContent(); + } catch (AmazonServiceException e) { + logger.error(e.getErrorMessage()); } - return inputstream; + return inputStream; } /** @@ -126,9 +203,9 @@ public InputStream getFileInputStream(Path file) { */ @Override public boolean isGzipped(Path file) throws IOException { - try (InputStream is = getFileInputStream(file)) { + try (InputStream inputStream = getFileInputStream(file)) { byte[] bytes = new byte[2]; - is.read(bytes); + inputStream.read(bytes); return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } @@ -150,23 +227,24 @@ public SequenceFile createSequenceFile(Path file) { return new CloudSequenceFile(file); } - /** - * Removes the leading "/" from the absolute path - * returns the rest of the path. - * - * @param file - * @return - */ - private String getAwsFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - return absolutePath; - } - /** * {@inheritDoc} */ @Override public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, + StandardOpenOption.WRITE)) { + try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { + for (long p = 0, l = in.size(); p < l; ) { + p += in.transferTo(p, l - p, out); + } + } catch (IOException e) { + throw new ConcatenateException("Could not open input file for reading", e); + } + + } catch (IOException e) { + throw new ConcatenateException("Could not open target file for writing", e); + } } /** @@ -175,6 +253,48 @@ public void appendToFile(Path target, SequenceFile file) throws ConcatenateExcep @Override public String getFileExtension(List toConcatenate) throws ConcatenateException { String selectedExtension = null; + for (SequencingObject object : toConcatenate) { + + for (SequenceFile file : object.getFiles()) { + String fileName = getFileName(file.getFile()); + + Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + .filter(e -> fileName.endsWith(e)) + .findFirst(); + + if (!currentExtensionOpt.isPresent()) { + throw new ConcatenateException("File extension is not valid " + fileName); + } + + String currentExtension = currentExtensionOpt.get(); + + if (selectedExtension == null) { + selectedExtension = currentExtensionOpt.get(); + } else if (selectedExtension != currentExtensionOpt.get()) { + throw new ConcatenateException( + "Extensions of files to concatenate do not match " + currentExtension + " vs " + + selectedExtension); + } + } + } + return selectedExtension; } + + /** + * Removes the leading "/" from the absolute path + * returns the rest of the path. + * + * @param file + * @return + */ + private String getAwsFileAbsolutePath(Path file) { + String absolutePath = file.toAbsolutePath().toString(); + if(absolutePath.charAt(0) == '/') { + absolutePath = file.toAbsolutePath() + .toString() + .substring(1); + } + return absolutePath; + } } From 5e2131d0b3a2438a5c971e29b60569848dff900d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 14 Apr 2020 17:15:28 -0500 Subject: [PATCH 024/655] FASTQC images are retrieved from local or cloud storage depending on which implementation of iridaFileStorageService is being used --- .../workflow/analysis/AnalysisFastQC.java | 16 ++++-- .../listeners/AnalysisFastQCListener.java | 28 ++++++++++ .../IridaFileStorageAwsServiceImpl.java | 15 ++++++ .../IridaFileStorageAzureServiceImpl.java | 53 ++++++++++++------- .../IridaFileStorageLocalServiceImpl.java | 15 ++++++ .../filesystem/IridaFileStorageService.java | 8 +++ 6 files changed, 114 insertions(+), 21 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java index e386081e7cf..b0b17e3da39 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java @@ -2,22 +2,26 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.AnalysisFastQCListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; import javax.persistence.*; import javax.validation.constraints.NotNull; import java.io.IOException; -import java.nio.file.Files; import java.util.HashMap; import java.util.Map; import java.util.Set; + /** * Specific implementation of {@link Analysis} for storing properties created by FastQC. */ @Entity @Table(name = "analysis_fastqc") +@EntityListeners({ AnalysisFastQCListener.class }) public class AnalysisFastQC extends Analysis { @NotNull @@ -43,6 +47,8 @@ public class AnalysisFastQC extends Analysis { @JoinTable(joinColumns = @JoinColumn(name = "analysis_fastqc_id")) private final Set overrepresentedSequences; + private static IridaFileStorageService iridaFileStorageService; + /** * Required for hibernate, should not be used anywhere else, so private. */ @@ -382,6 +388,10 @@ public Set getOverrepresentedSequences() { return ImmutableSet.copyOf(overrepresentedSequences); } + public void setIridaFileStorageService(IridaFileStorageService iridaFileStorageService) { + this.iridaFileStorageService = iridaFileStorageService; + } + /** * Read the bytes for a fastqc image * @@ -389,9 +399,9 @@ public Set getOverrepresentedSequences() { * @return the bytes for the file * @throws IOException if the file couldn't be read */ - private byte[] getBytesForFile(String key) throws IOException { + private byte[] getBytesForFile(String key) { AnalysisOutputFile chart = getAnalysisOutputFile(key); - byte[] bytes = Files.readAllBytes(chart.getFile()); + byte[] bytes = iridaFileStorageService.readAllBytes(chart.getFile()); return bytes; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java new file mode 100644 index 00000000000..4823bf3d1e7 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java @@ -0,0 +1,28 @@ +package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; + +import javax.persistence.PostLoad; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; + +import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + +@Component +public class AnalysisFastQCListener { + @Autowired + private IridaFileStorageService iridaFileStorageService; + + /** + * After the SequenceFile entity is loaded this method will provide + * the entity access to the iridaFileStorageService + * + * @param analysisFastQC The entity to provide the iridaFileStorageService to + */ + @PostLoad + public void afterAnalysisFastQcLoad(AnalysisFastQC analysisFastQC) { + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); + analysisFastQC.setIridaFileStorageService(iridaFileStorageService); + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index bf8d9c59f64..21425d5bafb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -281,6 +281,20 @@ public String getFileExtension(List toConcatenate) t return selectedExtension; } + /** + * {@inheritDoc} + */ + @Override + public byte[] readAllBytes(Path file) { + byte[] bytes = new byte[0]; + try { + bytes = getFileInputStream(file).readAllBytes(); + } catch (IOException e) { + logger.error("Unable to read file " + e); + } + return bytes; + } + /** * Removes the leading "/" from the absolute path * returns the rest of the path. @@ -297,4 +311,5 @@ private String getAwsFileAbsolutePath(Path file) { } return absolutePath; } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 08796c8d46c..9718e76904d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Date; @@ -68,7 +69,7 @@ public File getTemporaryFile(Path file) { blobClient.downloadToFile(filePath); fileToProcess = new File(filePath); } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); + logger.error("Couldn't find file [" + e + "]"); } return fileToProcess; @@ -85,7 +86,7 @@ public Long getFileSize(Path file) { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = blobClient.getProperties().getBlobSize(); } catch (BlobStorageException e) { - logger.trace("Couldn't calculate size as the file was not found [" + e + "]"); + logger.error("Couldn't calculate size as the file was not found [" + e + "]"); } return fileSize; } @@ -201,22 +202,6 @@ public SequenceFile createSequenceFile(Path file) { return new CloudSequenceFile(file); } - /** - * Removes the leading "/" from the absolute path - * returns the rest of the path. - * - * @param file - * @return - */ - private String getAzureFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - if(absolutePath.charAt(0) == '/') { - absolutePath = file.toAbsolutePath() - .toString() - .substring(1); - } - return absolutePath; - } /** * {@inheritDoc} @@ -271,4 +256,36 @@ public String getFileExtension(List toConcatenate) t return selectedExtension; } + + /** + * {@inheritDoc} + */ + @Override + public byte[] readAllBytes(Path file) { + byte[] bytes = new byte[0]; + try { + bytes = getFileInputStream(file).readAllBytes(); + } catch (IOException e) + { + logger.error("Unable to read file"); + } + return bytes; + } + + /** + * Removes the leading "/" from the absolute path + * returns the rest of the path. + * + * @param file + * @return + */ + private String getAzureFileAbsolutePath(Path file) { + String absolutePath = file.toAbsolutePath().toString(); + if(absolutePath.charAt(0) == '/') { + absolutePath = file.toAbsolutePath() + .toString() + .substring(1); + } + return absolutePath; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index 33209cead37..8a2249149b4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -231,4 +231,19 @@ public String getFileExtension(List toConcatenate) t return selectedExtension; } + + /** + * {@inheritDoc} + */ + @Override + public byte[] readAllBytes(Path file) { + byte[] bytes = new byte[0]; + try { + bytes = Files.readAllBytes(file); + } catch (IOException e) + { + logger.error("Unable to read file"); + } + return bytes; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index d0a50490c2d..2adca7ef29b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -144,4 +144,12 @@ public interface IridaFileStorageService { * @throws ConcatenateException if the files have different or invalid extensions */ public String getFileExtension(List toConcatenate) throws ConcatenateException; + + /** + * Read the bytes for an image + * + * @param file path to file which to read + * @return the bytes for the file + */ + public byte[] readAllBytes(Path file); } From cdd45c63fe3cc7d9bdc9fbd9c61c907149d3720f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 14 Apr 2020 17:59:19 -0500 Subject: [PATCH 025/655] Reference, sequence, and assembly can be downloaded from cloud as well as local storage --- .../repositories/filesystem/IridaFileStorageService.java | 1 + .../irida/ria/web/files/ReferenceFileController.java | 8 ++++++-- .../irida/ria/web/files/SequenceFileController.java | 7 +++++-- .../irida/ria/web/samples/SamplesController.java | 8 ++++++-- .../ria/unit/web/files/ReferenceFileControllerTest.java | 5 ++++- .../ria/unit/web/files/SequenceFileControllerTest.java | 5 ++++- .../irida/ria/unit/web/samples/SamplesControllerTest.java | 5 ++++- 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 2adca7ef29b..d9f91662634 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -152,4 +152,5 @@ public interface IridaFileStorageService { * @return the bytes for the file */ public byte[] readAllBytes(Path file); + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java index 8ff16343e33..1a60ea975a4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java @@ -31,6 +31,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.UnsupportedReferenceFileContentError; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; @@ -48,13 +49,16 @@ public class ReferenceFileController { private final ReferenceFileService referenceFileService; private final MessageSource messageSource; + private final IridaFileStorageService iridaFileStorageService; + @Autowired public ReferenceFileController(ProjectService projectService, ReferenceFileService referenceFileService, - MessageSource messageSource) { + MessageSource messageSource, IridaFileStorageService iridaFileStorageService) { this.projectService = projectService; this.referenceFileService = referenceFileService; this.messageSource = messageSource; this.dateFormatter = new DateFormatter(); + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -74,7 +78,7 @@ public void downloadReferenceFile(@PathVariable Long fileId, ReferenceFile file = referenceFileService.read(fileId); Path path = file.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getLabel() + "\""); - Files.copy(path, response.getOutputStream()); + iridaFileStorageService.getFileInputStream(path).transferTo(response.getOutputStream()); response.flushBuffer(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java index a0db8012d94..d0be0c8063d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java @@ -28,6 +28,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -63,13 +64,15 @@ public class SequenceFileController { */ private SequencingObjectService sequencingObjectService; private final AnalysisService analysisService; + private IridaFileStorageService iridaFileStorageService; @Autowired public SequenceFileController(SequencingObjectService sequencingObjectService, SequencingRunService sequencingRunService, - final AnalysisService analysisService) { + final AnalysisService analysisService, IridaFileStorageService iridaFileStorageService) { this.sequencingObjectService = sequencingObjectService; this.dateFormatter = new DateFormatter(); this.analysisService = analysisService; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -130,7 +133,7 @@ public void downloadSequenceFile(@PathVariable Long sequencingObjectId, @PathVar SequenceFile sequenceFile = sequencingObject.getFileWithId(sequenceFileId); Path path = sequenceFile.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + sequenceFile.getLabel() + "\""); - Files.copy(path, response.getOutputStream()); + iridaFileStorageService.getFileInputStream(path).transferTo(response.getOutputStream()); response.flushBuffer(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index dcf06e75b08..03c5cb590c5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -39,6 +39,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.User; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.web.BaseController; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleDetails; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; @@ -111,11 +112,13 @@ public class SamplesController extends BaseController { private final MessageSource messageSource; + private final IridaFileStorageService iridaFileStorageService; + @Autowired public SamplesController(SampleService sampleService, ProjectService projectService, SequencingObjectService sequencingObjectService, UpdateSamplePermission updateSamplePermission, MetadataTemplateService metadataTemplateService, GenomeAssemblyService genomeAssemblyService, - MessageSource messageSource) { + MessageSource messageSource, IridaFileStorageService iridaFileStorageService) { this.sampleService = sampleService; this.projectService = projectService; this.sequencingObjectService = sequencingObjectService; @@ -123,6 +126,7 @@ public SamplesController(SampleService sampleService, ProjectService projectServ this.metadataTemplateService = metadataTemplateService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; + this.iridaFileStorageService = iridaFileStorageService; } /************************************************************************************************ @@ -347,7 +351,7 @@ public void downloadAssembly(@PathVariable Long sampleId, @PathVariable Long ass Path path = genomeAssembly.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + genomeAssembly.getLabel() + "\""); - Files.copy(path, response.getOutputStream()); + iridaFileStorageService.getFileInputStream(path).transferTo(response.getOutputStream()); response.flushBuffer(); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java index 1619fed8fc0..41da238759f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java @@ -29,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ProjectReferenceFileJoin; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.files.ReferenceFileController; import ca.corefacility.bioinformatics.irida.service.ProjectService; @@ -55,19 +56,21 @@ public class ReferenceFileControllerTest { private ProjectService projectService; private ReferenceFileService referenceFileService; private MessageSource messageSource; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { projectService = mock(ProjectService.class); referenceFileService = mock(ReferenceFileService.class); messageSource = mock(MessageSource.class); + iridaFileStorageService = mock(IridaFileStorageService.class); // Set up the reference file Path path = Paths.get(FILE_PATH); ReferenceFile file = new ReferenceFile(path); when(referenceFileService.read(FILE_ID)).thenReturn(file); - controller = new ReferenceFileController(projectService, referenceFileService, messageSource); + controller = new ReferenceFileController(projectService, referenceFileService, messageSource, iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index d5ece606df7..d04542ef5ca 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -23,6 +23,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; @@ -43,13 +44,15 @@ public class SequenceFileControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; private AnalysisService analysisService; + private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { sequencingRunService = mock(SequencingRunService.class); analysisService = mock(AnalysisService.class); objectService = mock(SequencingObjectService.class); - controller = new SequenceFileController(objectService, sequencingRunService, analysisService); + iridaFileStorageService = mock(IridaFileStorageService.class); + controller = new SequenceFileController(objectService, sequencingRunService, analysisService, iridaFileStorageService); Path path = Paths.get(FILE_PATH); SequenceFile file = new LocalSequenceFile(path); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index ace2ef143bc..2ffe56d6597 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -58,6 +59,7 @@ public class SamplesControllerTest { private MetadataTemplateService metadataTemplateService; private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; + private IridaFileStorageService iridaFileStorageService; @Before @@ -70,8 +72,9 @@ public void setUp() { messageSource = mock(MessageSource.class); updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); + iridaFileStorageService = mock(IridaFileStorageService.class); controller = new SamplesController(sampleService, projectService, sequencingObjectService, - updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); + updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource, iridaFileStorageService); } // ************************************************************************************************ From 96988c29c7620a30dd059156a23aed3c79c213ce Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:18:14 -0500 Subject: [PATCH 026/655] Removed aws and azure implementations from the base and will be put into individual implementations --- .../services/IridaApiServicesConfig.java | 17 +- .../IridaFileStorageAwsServiceImpl.java | 180 ------------ .../IridaFileStorageAzureServiceImpl.java | 274 ------------------ 3 files changed, 1 insertion(+), 470 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 8a268114ab9..9667548da4f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -128,15 +128,6 @@ public class IridaApiServicesConfig { @Value("${locales.enabled}") private String availableLocales; - @Value("${irida.storage.type}") - private String storageType; - - @Value("${azure.container.name:#{null}}") - private String containerName; - - @Value("${azure.account.connection.string:#{null}}") - private String connectionStr; - @Autowired private IridaPluginConfig.IridaPluginList pipelinePlugins; @@ -297,13 +288,7 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageService") public IridaFileStorageService iridaFileStorageService() { - if(storageType.equalsIgnoreCase("azure")) { - return new IridaFileStorageAzureServiceImpl(connectionStr, containerName); - } else if (storageType.equalsIgnoreCase("aws")) { - return new IridaFileStorageAwsServiceImpl(); - } else { - return new IridaFileStorageLocalServiceImpl(); - } + return new IridaFileStorageLocalServiceImpl(); } @Bean(name = "uploadFileProcessingChain") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java deleted file mode 100644 index 064c8be5e34..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ /dev/null @@ -1,180 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.filesystem; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.util.List; -import java.util.zip.GZIPInputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; - - -/** - * Component implementation of file utitlities for aws storage - */ -@Component -public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsServiceImpl.class); - - @Autowired - public IridaFileStorageAwsServiceImpl(){ - } - - /** - * {@inheritDoc} - */ - @Override - public File getTemporaryFile(Path file) { - File fileToProcess = null; - - // Implement AWS code to get file - - return fileToProcess; - } - - /** - * {@inheritDoc} - */ - @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; - // Implement AWS code to get file size - return fileSize; - } - - /** - * {@inheritDoc} - */ - @Override - public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { - // Implement AWS code to upload file to bucket - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFiles() { - } - - /** - * {@inheritDoc} - */ - @Override - public boolean storageTypeIsLocal(){ - return false; - } - - /** - * {@inheritDoc} - */ - public String getFileName(Path file) { - String fileName = ""; - // Implement AWS code to get file name - return fileName; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean fileExists(Path file) { - return false; - } - - /** - * {@inheritDoc} - */ - @Override - public InputStream getFileInputStream(Path file) { - InputStream inputstream = null; - try { - inputstream = new FileInputStream(file.toString()); - - } catch(IOException e) { - - } - return inputstream; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isGzipped(Path file) throws IOException { - try (InputStream is = getFileInputStream(file)) { - byte[] bytes = new byte[2]; - is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) - && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); - } - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createEmptySequenceFile() { - return new CloudSequenceFile(); - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createSequenceFile(Path file) { - return new CloudSequenceFile(file); - } - - /** - * Removes the leading "/" from the absolute path - * returns the rest of the path. - * - * @param file - * @return - */ - private String getAwsFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - return absolutePath; - } - - /** - * {@inheritDoc} - */ - @Override - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { - } - - /** - * {@inheritDoc} - */ - @Override - public String getFileExtension(List toConcatenate) throws ConcatenateException { - String selectedExtension = null; - return selectedExtension; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java deleted file mode 100644 index 08796c8d46c..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ /dev/null @@ -1,274 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.filesystem; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.zip.GZIPInputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobStorageException; - -/** - * Component implementation of file utitlities for azure storage - */ -@Component -public class IridaFileStorageAzureServiceImpl implements IridaFileStorageService { - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); - - private BlobServiceClient blobServiceClient; - private BlobContainerClient containerClient ; - private BlobClient blobClient; - - @Autowired - public IridaFileStorageAzureServiceImpl(String connectionStr, String containerName){ - this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) - .buildClient(); - this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - } - - /** - * {@inheritDoc} - */ - @Override - public File getTemporaryFile(Path file) { - File fileToProcess = null; - - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - - try { - // Create a file that will be unique in the /tmp/ folder. We append the current date/time - // to the file name - String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String [] blobNameTokens = blobClient.getBlobName().split("/"); - String fileName = blobNameTokens[blobNameTokens.length-1]; - String filePath = tmpDir + fileName; - blobClient.downloadToFile(filePath); - fileToProcess = new File(filePath); - } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); - } - - return fileToProcess; - } - - /** - * {@inheritDoc} - */ - @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; - try { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - fileSize = blobClient.getProperties().getBlobSize(); - } catch (BlobStorageException e) { - logger.trace("Couldn't calculate size as the file was not found [" + e + "]"); - } - return fileSize; - } - - /** - * {@inheritDoc} - */ - @Override - public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); - - logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); - blobClient.uploadFromFile(source.toString(), false); - logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFiles() { - } - - /** - * {@inheritDoc} - */ - @Override - public boolean storageTypeIsLocal(){ - return false; - } - - /** - * {@inheritDoc} - */ - public String getFileName(Path file) { - String fileName = ""; - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - try { - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String[] blobNameTokens = blobClient.getBlobName() - .split("/"); - fileName = blobNameTokens[blobNameTokens.length - 1]; - } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); - } - - return fileName; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean fileExists(Path file) { - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - if(blobClient.getProperties().getBlobSize() > 0) { - return true; - } - return false; - } - - /** - * {@inheritDoc} - */ - @Override - public InputStream getFileInputStream(Path file) { - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - return blobClient.openInputStream(); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isGzipped(Path file) throws IOException { - try (InputStream is = getFileInputStream(file)) { - byte[] bytes = new byte[2]; - is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) - && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); - } - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createEmptySequenceFile() { - return new CloudSequenceFile(); - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createSequenceFile(Path file) { - return new CloudSequenceFile(file); - } - - /** - * Removes the leading "/" from the absolute path - * returns the rest of the path. - * - * @param file - * @return - */ - private String getAzureFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - if(absolutePath.charAt(0) == '/') { - absolutePath = file.toAbsolutePath() - .toString() - .substring(1); - } - return absolutePath; - } - - /** - * {@inheritDoc} - */ - @Override - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { - try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, - StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { - for (long p = 0, l = in.size(); p < l; ) { - p += in.transferTo(p, l - p, out); - } - } catch (IOException e) { - throw new ConcatenateException("Could not open input file for reading", e); - } - - } catch (IOException e) { - throw new ConcatenateException("Could not open target file for writing", e); - } - } - - /** - * {@inheritDoc} - */ - @Override - public String getFileExtension(List toConcatenate) throws ConcatenateException { - String selectedExtension = null; - for (SequencingObject object : toConcatenate) { - - for (SequenceFile file : object.getFiles()) { - String fileName = getFileName(file.getFile()); - - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() - .filter(e -> fileName.endsWith(e)) - .findFirst(); - - if (!currentExtensionOpt.isPresent()) { - throw new ConcatenateException("File extension is not valid " + fileName); - } - - String currentExtension = currentExtensionOpt.get(); - - if (selectedExtension == null) { - selectedExtension = currentExtensionOpt.get(); - } else if (selectedExtension != currentExtensionOpt.get()) { - throw new ConcatenateException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " - + selectedExtension); - } - } - } - - return selectedExtension; - } -} From 0445c89578747aac1142ce23e532c5edcc401053 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:21:44 -0500 Subject: [PATCH 027/655] Added azure file storage implementation --- .../services/IridaApiServicesConfig.java | 15 +- .../IridaFileStorageAzureServiceImpl.java | 274 ++++++++++++++++++ 2 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 9667548da4f..4d7bdb00fa8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -128,6 +128,15 @@ public class IridaApiServicesConfig { @Value("${locales.enabled}") private String availableLocales; + @Value("${irida.storage.type}") + private String storageType; + + @Value("${azure.container.name:#{null}}") + private String containerName; + + @Value("${azure.account.connection.string:#{null}}") + private String connectionStr; + @Autowired private IridaPluginConfig.IridaPluginList pipelinePlugins; @@ -288,7 +297,11 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageService") public IridaFileStorageService iridaFileStorageService() { - return new IridaFileStorageLocalServiceImpl(); + if(storageType.equalsIgnoreCase("azure")) { + return new IridaFileStorageAzureServiceImpl(connectionStr, containerName); + } else { + return new IridaFileStorageLocalServiceImpl(); + } } @Bean(name = "uploadFileProcessingChain") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java new file mode 100644 index 00000000000..08796c8d46c --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -0,0 +1,274 @@ +package ca.corefacility.bioinformatics.irida.repositories.filesystem; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.zip.GZIPInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobStorageException; + +/** + * Component implementation of file utitlities for azure storage + */ +@Component +public class IridaFileStorageAzureServiceImpl implements IridaFileStorageService { + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); + + private BlobServiceClient blobServiceClient; + private BlobContainerClient containerClient ; + private BlobClient blobClient; + + @Autowired + public IridaFileStorageAzureServiceImpl(String connectionStr, String containerName){ + this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) + .buildClient(); + this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + } + + /** + * {@inheritDoc} + */ + @Override + public File getTemporaryFile(Path file) { + File fileToProcess = null; + + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + + try { + // Create a file that will be unique in the /tmp/ folder. We append the current date/time + // to the file name + String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. + String [] blobNameTokens = blobClient.getBlobName().split("/"); + String fileName = blobNameTokens[blobNameTokens.length-1]; + String filePath = tmpDir + fileName; + blobClient.downloadToFile(filePath); + fileToProcess = new File(filePath); + } catch (BlobStorageException e) { + logger.trace("Couldn't find file [" + e + "]"); + } + + return fileToProcess; + } + + /** + * {@inheritDoc} + */ + @Override + public Long getFileSize(Path file) { + Long fileSize = 0L; + try { + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + fileSize = blobClient.getProperties().getBlobSize(); + } catch (BlobStorageException e) { + logger.trace("Couldn't calculate size as the file was not found [" + e + "]"); + } + return fileSize; + } + + /** + * {@inheritDoc} + */ + @Override + public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { + // We set the blobClient "path" to which we want to upload our file to + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); + + logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); + blobClient.uploadFromFile(source.toString(), false); + logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFile() { + } + + /** + * {@inheritDoc} + */ + @Override + public void downloadFiles() { + } + + /** + * {@inheritDoc} + */ + @Override + public boolean storageTypeIsLocal(){ + return false; + } + + /** + * {@inheritDoc} + */ + public String getFileName(Path file) { + String fileName = ""; + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + try { + // Since the file system is virtual the full file path is the file name. + // We split it on "/" and get the last token which is the actual file name. + String[] blobNameTokens = blobClient.getBlobName() + .split("/"); + fileName = blobNameTokens[blobNameTokens.length - 1]; + } catch (BlobStorageException e) { + logger.trace("Couldn't find file [" + e + "]"); + } + + return fileName; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean fileExists(Path file) { + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + if(blobClient.getProperties().getBlobSize() > 0) { + return true; + } + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public InputStream getFileInputStream(Path file) { + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + return blobClient.openInputStream(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isGzipped(Path file) throws IOException { + try (InputStream is = getFileInputStream(file)) { + byte[] bytes = new byte[2]; + is.read(bytes); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + } + } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createEmptySequenceFile() { + return new CloudSequenceFile(); + } + + /** + * {@inheritDoc} + */ + @Override + public SequenceFile createSequenceFile(Path file) { + return new CloudSequenceFile(file); + } + + /** + * Removes the leading "/" from the absolute path + * returns the rest of the path. + * + * @param file + * @return + */ + private String getAzureFileAbsolutePath(Path file) { + String absolutePath = file.toAbsolutePath().toString(); + if(absolutePath.charAt(0) == '/') { + absolutePath = file.toAbsolutePath() + .toString() + .substring(1); + } + return absolutePath; + } + + /** + * {@inheritDoc} + */ + @Override + public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, + StandardOpenOption.WRITE)) { + try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { + for (long p = 0, l = in.size(); p < l; ) { + p += in.transferTo(p, l - p, out); + } + } catch (IOException e) { + throw new ConcatenateException("Could not open input file for reading", e); + } + + } catch (IOException e) { + throw new ConcatenateException("Could not open target file for writing", e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public String getFileExtension(List toConcatenate) throws ConcatenateException { + String selectedExtension = null; + for (SequencingObject object : toConcatenate) { + + for (SequenceFile file : object.getFiles()) { + String fileName = getFileName(file.getFile()); + + Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + .filter(e -> fileName.endsWith(e)) + .findFirst(); + + if (!currentExtensionOpt.isPresent()) { + throw new ConcatenateException("File extension is not valid " + fileName); + } + + String currentExtension = currentExtensionOpt.get(); + + if (selectedExtension == null) { + selectedExtension = currentExtensionOpt.get(); + } else if (selectedExtension != currentExtensionOpt.get()) { + throw new ConcatenateException( + "Extensions of files to concatenate do not match " + currentExtension + " vs " + + selectedExtension); + } + } + } + + return selectedExtension; + } +} From cb0c0ce9d1e72d2b2becdc756e2140eb4720d3ea Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:24:12 -0500 Subject: [PATCH 028/655] Removed azure implementation from this branch as it is in it's own branch --- .../services/IridaApiServicesConfig.java | 12 +- .../IridaFileStorageAzureServiceImpl.java | 274 ------------------ 2 files changed, 1 insertion(+), 285 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 289658aef25..fcb3504fec2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -128,14 +128,6 @@ public class IridaApiServicesConfig { @Value("${locales.enabled}") private String availableLocales; - @Value("${irida.storage.type}") - private String storageType; - - @Value("${azure.container.name:#{null}}") - private String containerName; - - @Value("${azure.account.connection.string:#{null}}") - private String connectionStr; @Value("${aws.bucket.name:#{null}}") private String awsBucketName; @@ -310,9 +302,7 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageService") public IridaFileStorageService iridaFileStorageService() { - if(storageType.equalsIgnoreCase("azure")) { - return new IridaFileStorageAzureServiceImpl(connectionStr, containerName); - } else if (storageType.equalsIgnoreCase("aws")) { + if (storageType.equalsIgnoreCase("aws")) { return new IridaFileStorageAwsServiceImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); } else { return new IridaFileStorageLocalServiceImpl(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java deleted file mode 100644 index 08796c8d46c..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ /dev/null @@ -1,274 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.filesystem; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.zip.GZIPInputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.CloudSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobStorageException; - -/** - * Component implementation of file utitlities for azure storage - */ -@Component -public class IridaFileStorageAzureServiceImpl implements IridaFileStorageService { - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); - - private BlobServiceClient blobServiceClient; - private BlobContainerClient containerClient ; - private BlobClient blobClient; - - @Autowired - public IridaFileStorageAzureServiceImpl(String connectionStr, String containerName){ - this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) - .buildClient(); - this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - } - - /** - * {@inheritDoc} - */ - @Override - public File getTemporaryFile(Path file) { - File fileToProcess = null; - - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - - try { - // Create a file that will be unique in the /tmp/ folder. We append the current date/time - // to the file name - String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String [] blobNameTokens = blobClient.getBlobName().split("/"); - String fileName = blobNameTokens[blobNameTokens.length-1]; - String filePath = tmpDir + fileName; - blobClient.downloadToFile(filePath); - fileToProcess = new File(filePath); - } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); - } - - return fileToProcess; - } - - /** - * {@inheritDoc} - */ - @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; - try { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - fileSize = blobClient.getProperties().getBlobSize(); - } catch (BlobStorageException e) { - logger.trace("Couldn't calculate size as the file was not found [" + e + "]"); - } - return fileSize; - } - - /** - * {@inheritDoc} - */ - @Override - public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { - // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); - - logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); - blobClient.uploadFromFile(source.toString(), false); - logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFile() { - } - - /** - * {@inheritDoc} - */ - @Override - public void downloadFiles() { - } - - /** - * {@inheritDoc} - */ - @Override - public boolean storageTypeIsLocal(){ - return false; - } - - /** - * {@inheritDoc} - */ - public String getFileName(Path file) { - String fileName = ""; - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - try { - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String[] blobNameTokens = blobClient.getBlobName() - .split("/"); - fileName = blobNameTokens[blobNameTokens.length - 1]; - } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); - } - - return fileName; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean fileExists(Path file) { - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - if(blobClient.getProperties().getBlobSize() > 0) { - return true; - } - return false; - } - - /** - * {@inheritDoc} - */ - @Override - public InputStream getFileInputStream(Path file) { - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - return blobClient.openInputStream(); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isGzipped(Path file) throws IOException { - try (InputStream is = getFileInputStream(file)) { - byte[] bytes = new byte[2]; - is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) - && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); - } - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createEmptySequenceFile() { - return new CloudSequenceFile(); - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createSequenceFile(Path file) { - return new CloudSequenceFile(file); - } - - /** - * Removes the leading "/" from the absolute path - * returns the rest of the path. - * - * @param file - * @return - */ - private String getAzureFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - if(absolutePath.charAt(0) == '/') { - absolutePath = file.toAbsolutePath() - .toString() - .substring(1); - } - return absolutePath; - } - - /** - * {@inheritDoc} - */ - @Override - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { - try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, - StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { - for (long p = 0, l = in.size(); p < l; ) { - p += in.transferTo(p, l - p, out); - } - } catch (IOException e) { - throw new ConcatenateException("Could not open input file for reading", e); - } - - } catch (IOException e) { - throw new ConcatenateException("Could not open target file for writing", e); - } - } - - /** - * {@inheritDoc} - */ - @Override - public String getFileExtension(List toConcatenate) throws ConcatenateException { - String selectedExtension = null; - for (SequencingObject object : toConcatenate) { - - for (SequenceFile file : object.getFiles()) { - String fileName = getFileName(file.getFile()); - - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() - .filter(e -> fileName.endsWith(e)) - .findFirst(); - - if (!currentExtensionOpt.isPresent()) { - throw new ConcatenateException("File extension is not valid " + fileName); - } - - String currentExtension = currentExtensionOpt.get(); - - if (selectedExtension == null) { - selectedExtension = currentExtensionOpt.get(); - } else if (selectedExtension != currentExtensionOpt.get()) { - throw new ConcatenateException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " - + selectedExtension); - } - } - } - - return selectedExtension; - } -} From ba77462890a28e8eeb17db2571fd7b8ccdfc8816 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:29:18 -0500 Subject: [PATCH 029/655] Removed aws and azure dependencies from pom as they are in separate branches --- pom.xml | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 9d1e527d4fc..2417822bdd8 100644 --- a/pom.xml +++ b/pom.xml @@ -344,11 +344,6 @@ jackson-databind ${jackson.version} - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -677,35 +672,6 @@ gson ${com.google.code.gson.version} - - - com.azure - azure-storage-blob - 12.0.0 - - - jakarta.xml.bind - jakarta.xml.bind-api - - - jakarta.activation - jakarta.activation-api - - - - - - com.amazonaws - aws-java-sdk-s3 - 1.11.376 - - - commons-logging - commons-logging - - - - com.monitorjbl @@ -1168,7 +1134,7 @@ 1.4.01 3.5.1 3.0 - 2.10.3 + 2.9.10 2.3.4 1.3.1 3.0.0 From 74a8321c54e160dd9e90d0e8334bd71a1592618e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:32:33 -0500 Subject: [PATCH 030/655] Removed aws dependencies --- pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pom.xml b/pom.xml index 9d1e527d4fc..824037432fe 100644 --- a/pom.xml +++ b/pom.xml @@ -694,18 +694,6 @@ - - com.amazonaws - aws-java-sdk-s3 - 1.11.376 - - - commons-logging - commons-logging - - - - com.monitorjbl From 38ca693a0813ffa9c6565c0c0b3de137c420524b Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:34:31 -0500 Subject: [PATCH 031/655] Removed azure dependencies --- pom.xml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/pom.xml b/pom.xml index 1d3a4a27c5a..fa36a5e5445 100644 --- a/pom.xml +++ b/pom.xml @@ -344,11 +344,6 @@ jackson-databind ${jackson.version} - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -678,22 +673,6 @@ ${com.google.code.gson.version} - - com.azure - azure-storage-blob - ${azure-storage-blob.version} - - - jakarta.xml.bind - jakarta.xml.bind-api - - - jakarta.activation - jakarta.activation-api - - - - com.amazonaws aws-java-sdk-s3 @@ -1182,7 +1161,7 @@ 1.4.01 3.5.1 3.0 - 2.10.3 + 2.9.10 2.3.4 1.3.1 3.0.0 From 163cafa7a846102905b39d5bf4f6bcf45af36015 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 16:36:13 -0500 Subject: [PATCH 032/655] Added newline --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 2417822bdd8..dd69f67cd0e 100644 --- a/pom.xml +++ b/pom.xml @@ -678,6 +678,7 @@ xlsx-streamer ${com.monitorjbl.xml.streamer.version} + From 4ebd4a0881382a834b289c009e166d9b4fb4160d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 17:20:29 -0500 Subject: [PATCH 033/655] Reverted tests as they are in another branch --- .../irida/events/ProjectEventHandlerTest.java | 7 +- .../unit/SequenceFilePairTest.java | 9 +- .../unit/AnalysisSubmissionTest.java | 3 +- ...quencingObjectConcatenatorFactoryTest.java | 28 +---- .../SequenceFilePairConcatenatorTest.java | 16 +-- ...SingleEndSequenceFileConcatenatorTest.java | 18 +-- .../AutomatedAnalysisFileProcessorTest.java | 14 +-- .../impl/unit/ChecksumFileProcessorTest.java | 20 +--- .../impl/unit/CoverageFileProcessorTest.java | 7 +- .../unit/DefaultFileProcessingChainTest.java | 15 +-- .../impl/unit/FastqcFileProcessorTest.java | 26 +---- .../impl/unit/GzipFileProcessorTest.java | 25 +--- .../SequenceFileRepositoryImplTest.java | 29 ++--- .../irida/ria/unit/TestDataFactory.java | 6 +- .../unit/web/SequencingRunControllerTest.java | 4 +- .../web/files/SequenceFileControllerTest.java | 3 +- .../ProjectSamplesControllerTest.java | 5 +- .../web/samples/SamplesControllerTest.java | 15 ++- .../ReadAnalysisSubmissionPermissionTest.java | 8 +- .../service/DatabaseSetupGalaxyITService.java | 13 ++- .../AnalysisExecutionServiceGalaxyTest.java | 4 +- .../AnalysisWorkspaceServiceGalaxyIT.java | 7 +- .../AnalysisWorkspaceServiceGalaxyTest.java | 11 +- .../export/ExportUploadServiceTest.java | 3 +- .../NcbiExportSubmissionServiceTest.java | 5 +- .../SequencingObjectServiceImplIT.java | 21 ++-- .../SequencingRunServiceImplIT.java | 6 +- .../unit/sample/SampleServiceImplTest.java | 17 ++- ...eEndSequenceFileRemoteServiceImplTest.java | 4 +- .../controller/test/unit/TestDataFactory.java | 108 +++++++++--------- 30 files changed, 177 insertions(+), 280 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java index de1e20af6c9..4b7d43673d6 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/events/ProjectEventHandlerTest.java @@ -26,7 +26,6 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.User; @@ -122,7 +121,7 @@ public void testHandleSequenceFileAddedEventSingle() { Class clazz = DataAddedToSampleProjectEvent.class; Project project = new Project(); Sample sample = new Sample(); - SequenceFile file = new LocalSequenceFile(); + SequenceFile file = new SequenceFile(); SingleEndSequenceFile seqObj = new SingleEndSequenceFile(file); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(sample, seqObj); @@ -152,7 +151,7 @@ public void testHandleSequenceFileAddedEventMultipleReturn() { Class clazz = DataAddedToSampleProjectEvent.class; Project project = new Project(); Sample sample = new Sample(); - SequenceFile file = new LocalSequenceFile(); + SequenceFile file = new SequenceFile(); SingleEndSequenceFile seqObj1 = new SingleEndSequenceFile(file); SingleEndSequenceFile seqObj2 = new SingleEndSequenceFile(file); SampleSequencingObjectJoin join1 = new SampleSequencingObjectJoin(sample, seqObj1); @@ -185,7 +184,7 @@ public void testHandleSequenceFileAddedEventMultipleProjects() { Project project = new Project("p1"); Project project2 = new Project("p2"); Sample sample = new Sample(); - SequenceFile file = new LocalSequenceFile(); + SequenceFile file = new SequenceFile(); SingleEndSequenceFile seqObj = new SingleEndSequenceFile(file); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(sample, seqObj); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java index 8a1da220717..1672858ca34 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java @@ -10,7 +10,6 @@ import org.junit.Before; import org.junit.Test; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; @@ -50,11 +49,11 @@ public void setup() throws IOException { reversePathGood = tempDir.resolve("Test_R2_001.fastq"); reversePathBad = tempDir.resolve("Test_B.fastq"); - sequenceFileForwardGood = new LocalSequenceFile(forwardPathGood); - sequenceFileForwardBad = new LocalSequenceFile(forwardPathBad); + sequenceFileForwardGood = new SequenceFile(forwardPathGood); + sequenceFileForwardBad = new SequenceFile(forwardPathBad); - sequenceFileReverseGood = new LocalSequenceFile(reversePathGood); - sequenceFileReverseBad = new LocalSequenceFile(reversePathBad); + sequenceFileReverseGood = new SequenceFile(reversePathGood); + sequenceFileReverseBad = new SequenceFile(reversePathBad); sequenceFilePairGood = new SequenceFilePair(sequenceFileForwardGood, sequenceFileReverseGood); sequenceFilePairBad = new SequenceFilePair(sequenceFileForwardBad, sequenceFileReverseBad); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java index 4290f702f38..b13f849c539 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/unit/AnalysisSubmissionTest.java @@ -11,7 +11,6 @@ import com.google.common.collect.Sets; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -24,7 +23,7 @@ */ public class AnalysisSubmissionTest { - private static final SequenceFile sequenceFile = new LocalSequenceFile(); + private static final SequenceFile sequenceFile = new SequenceFile(); private static final SingleEndSequenceFile singleEndFile = new SingleEndSequenceFile(sequenceFile); private static final ReferenceFile referenceFile = new ReferenceFile(); private static final Map inputParameters = ImmutableMap.of("test", "test"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index 560048aca52..96eb4f1893e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -1,20 +1,12 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; - import com.google.common.collect.Sets; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.util.Set; @@ -23,51 +15,43 @@ /** * Unit test for {@link SequencingObjectConcatenatorFactory} */ - -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequencingObjectConcatenatorFactoryTest { - - @Autowired - private IridaFileStorageService iridaFileStorageService; - @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class, iridaFileStorageService); + SingleEndSequenceFile.class); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class, iridaFileStorageService); + SequenceFilePair.class); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageService); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index b82eaad5f0c..a15a25a024a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -1,12 +1,8 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; - import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; @@ -23,22 +19,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequenceFilePairConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" + SEQUENCE + "\n+\n?????????").getBytes(); - private SequenceFilePairConcatenator concat; - - @Autowired - private IridaFileStorageService iridaFileStorageService; + SequenceFilePairConcatenator concat; @Before public void setUp() { - concat = new SequenceFilePairConcatenator(iridaFileStorageService); + concat = new SequenceFilePairConcatenator(); } @Test @@ -78,6 +68,6 @@ private SequenceFile createSequenceFile(String name) throws IOException { Path sequenceFile = Files.createTempFile(name, ".fastq"); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - return new LocalSequenceFile(sequenceFile); + return new SequenceFile(sequenceFile); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 8e3fef09d8b..9401f97ffa0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -1,20 +1,12 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -23,9 +15,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SingleEndSequenceFileConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" @@ -33,12 +22,9 @@ public class SingleEndSequenceFileConcatenatorTest { SingleEndSequenceFileConcatenator concat; - @Autowired - private IridaFileStorageService iridaFileStorageService; - @Before public void setUp() { - concat = new SingleEndSequenceFileConcatenator(iridaFileStorageService); + concat = new SingleEndSequenceFileConcatenator(); } @Test @@ -125,6 +111,6 @@ private SequenceFile createSequenceFile(String name, String extension) throws IO Path sequenceFile = Files.createTempFile(name, extension); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - return new LocalSequenceFile(sequenceFile); + return new SequenceFile(sequenceFile); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java index 779a3877604..234a413517b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/AutomatedAnalysisFileProcessorTest.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.user.User; @@ -111,8 +111,8 @@ public void setUp() throws IridaWorkflowNotFoundException { @Test public void testAssembleFile() { Long sequenceFileId = 1L; - SequenceFilePair pair = new SequenceFilePair(new LocalSequenceFile(Paths.get("file_R1_1.fastq.gz")), - new LocalSequenceFile(Paths.get("file_R2_1.fastq.gz"))); + SequenceFilePair pair = new SequenceFilePair(new SequenceFile(Paths.get("file_R1_1.fastq.gz")), + new SequenceFile(Paths.get("file_R2_1.fastq.gz"))); Sample sample = new Sample(); Project project = new Project(); @@ -152,8 +152,8 @@ public void testAssembleFile() { @Test public void testOtherWorkflow() { Long sequenceFileId = 1L; - SequenceFilePair pair = new SequenceFilePair(new LocalSequenceFile(Paths.get("file_R1_1.fastq.gz")), - new LocalSequenceFile(Paths.get("file_R2_1.fastq.gz"))); + SequenceFilePair pair = new SequenceFilePair(new SequenceFile(Paths.get("file_R1_1.fastq.gz")), + new SequenceFile(Paths.get("file_R2_1.fastq.gz"))); Sample sample = new Sample(); Project project = new Project(); @@ -203,8 +203,8 @@ public void testNoSubmissions() { @Test public void testOneProjectEnabled() { - SequenceFilePair pair = new SequenceFilePair(new LocalSequenceFile(Paths.get("file_R1_1.fastq.gz")), - new LocalSequenceFile(Paths.get("file_R2_1.fastq.gz"))); + SequenceFilePair pair = new SequenceFilePair(new SequenceFile(Paths.get("file_R1_1.fastq.gz")), + new SequenceFile(Paths.get("file_R2_1.fastq.gz"))); Sample sample = new Sample(); Project project = new Project("assemble me"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index 8e37c174ecb..ab25ff88a07 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -11,38 +11,24 @@ import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.ChecksumFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class ChecksumFileProcessorTest { private ChecksumFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; private static final String CHECKSUM = "aeaa0755dc44b393ffe12f02e9bd42b0169b12ca9c15708085db6a4ac9110ee0"; - @Autowired - private IridaFileStorageService iridaFileStorageService; - @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageService); + fileProcessor = new ChecksumFileProcessor(sequenceFileRepository); } @Test @@ -63,7 +49,7 @@ public void testChecksumCreated() throws IOException { @Test(expected = FileProcessorException.class) public void testFileNotExists() throws IOException { - final SequenceFile sf = new LocalSequenceFile(Paths.get("/reallyfakefile")); + final SequenceFile sf = new SequenceFile(Paths.get("/reallyfakefile")); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -71,7 +57,7 @@ public void testFileNotExists() throws IOException { } private SequenceFile constructSequenceFile() throws IOException { - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); Path sequenceFile = Files.createTempFile(null, null); Files.write(sequenceFile, FILE_CONTENTS.getBytes()); sf.setFile(sequenceFile); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java index ad6cbf1488b..8fb66583964 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/CoverageFileProcessorTest.java @@ -17,7 +17,6 @@ import ca.corefacility.bioinformatics.irida.model.sample.CoverageQCEntry; import ca.corefacility.bioinformatics.irida.model.sample.QCEntry; import ca.corefacility.bioinformatics.irida.model.sample.QCEntry.QCEntryStatus; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -45,7 +44,7 @@ public void testGoodCoverage() { Project p = new Project(); p.setGenomeSize(100L); p.setMinimumCoverage(2); - SequenceFile file = new LocalSequenceFile(); + SequenceFile file = new SequenceFile(); SequencingObject o = new SingleEndSequenceFile(file); AnalysisFastQC fqc = mock(AnalysisFastQC.class); Long baseCount = 300L; @@ -72,7 +71,7 @@ public void testBadCoverage() { Project p = new Project(); p.setGenomeSize(100L); p.setMinimumCoverage(5); - SequenceFile file = new LocalSequenceFile(); + SequenceFile file = new SequenceFile(); SequencingObject o = new SingleEndSequenceFile(file); AnalysisFastQC fqc = mock(AnalysisFastQC.class); Long baseCount = 300L; @@ -99,7 +98,7 @@ public void testRemoveExistingEntry() { Project p = new Project(); p.setGenomeSize(100L); p.setMinimumCoverage(2); - SequenceFile file = new LocalSequenceFile(); + SequenceFile file = new SequenceFile(); SequencingObject o = new SingleEndSequenceFile(file); AnalysisFastQC fqc = mock(AnalysisFastQC.class); Long baseCount = 300L; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java index ba8adc740b0..3e76018557d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java @@ -26,7 +26,6 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.DefaultFileProcessingChain; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -39,7 +38,6 @@ public class DefaultFileProcessingChainTest { private SequencingObjectRepository objectRepository; private QCEntryRepository qcRepository; - private IridaFileStorageService iridaFileStorageService; private SequencingObject seqObject; private Long objectId = 1L; @@ -48,7 +46,6 @@ public class DefaultFileProcessingChainTest { public void setUp() { this.objectRepository = mock(SequencingObjectRepository.class); this.qcRepository = mock(QCEntryRepository.class); - this.iridaFileStorageService = mock(IridaFileStorageService.class); seqObject = new NoFileSequencingObject(); when(objectRepository.findById(objectId)).thenReturn(Optional.of(seqObject)); @@ -56,7 +53,7 @@ public void setUp() { @Test(expected = FileProcessorTimeoutException.class) public void testExceedsTimeout() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); fileProcessingChain.setTimeout(1); fileProcessingChain.setSleepDuration(0); @@ -65,7 +62,7 @@ public void testExceedsTimeout() throws FileProcessorTimeoutException { @Test public void testProcessEmptyChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); when(objectRepository.existsById(objectId)).thenReturn(true); fileProcessingChain.launchChain(objectId); @@ -73,7 +70,7 @@ public void testProcessEmptyChain() throws FileProcessorTimeoutException { @Test public void testFailWithContinueChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -87,7 +84,7 @@ public void testFailWithContinueChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFastFailProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -99,7 +96,7 @@ public void testFastFailProcessorChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFailOnProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -109,7 +106,7 @@ public void testFailOnProcessorChain() throws FileProcessorTimeoutException { @Test public void testFailWriteQCEntry() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 3a306993f82..a244420e8f9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -16,26 +16,14 @@ import java.nio.file.Path; import java.util.Iterator; -import ca.corefacility.bioinformatics.irida.config.data.IridaApiJdbcDataSourceConfig; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; -import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestExecutionListeners; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; import org.springframework.util.ReflectionUtils; import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; @@ -44,8 +32,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import com.github.springtestdbunit.DbUnitTestExecutionListener; @@ -55,18 +41,12 @@ * * */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class FastqcFileProcessorTest { private FastqcFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private AnalysisOutputFileRepository outputFileRepository; private MessageSource messageSource; - @Autowired - private IridaFileStorageService iridaFileStorageService; - private static final Logger logger = LoggerFactory.getLogger(FastqcFileProcessorTest.class); private static final String SEQUENCE = "ACGTACGTN"; @@ -79,7 +59,7 @@ public void setUp() { messageSource = mock(MessageSource.class); sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); - fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageService); + fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository); } @Test(expected = FileProcessorException.class) @@ -88,7 +68,7 @@ public void testHandleFastaFile() throws IOException { // dummy), but that's A-OK. Path fasta = Files.createTempFile(null, null); Files.write(fasta, FASTA_FILE_CONTENTS.getBytes()); - SequenceFile sf = new LocalSequenceFile(fasta); + SequenceFile sf = new SequenceFile(fasta); sf.setId(1L); Runtime.getRuntime().addShutdownHook(new DeleteFileOnExit(fasta)); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -105,7 +85,7 @@ public void testHandleFastqFile() throws IOException, IllegalArgumentException, ArgumentCaptor argument = ArgumentCaptor.forClass(SequenceFile.class); - SequenceFile sf = new LocalSequenceFile(fastq); + SequenceFile sf = new SequenceFile(fastq); sf.setId(1L); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); try { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 1d401933eb4..8edbeea806c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -16,20 +16,12 @@ import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -37,23 +29,16 @@ * * */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; - @Autowired - private IridaFileStorageService iridaFileStorageService; - - @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageService); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE); } @Test(expected = FileProcessorException.class) @@ -75,7 +60,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageService); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference @@ -116,7 +101,7 @@ public void handleCompressedFileWithGzExtension() throws IOException { // the file processor should decompress the file, then update the // sequence file in the database. SequenceFile sf = constructSequenceFile(); - SequenceFile sfUpdated = new LocalSequenceFile(); + SequenceFile sfUpdated = new SequenceFile(); sfUpdated.setFile(sf.getFile()); final Long id = 1L; sf.setId(id); @@ -153,7 +138,7 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { // the file processor should decompress the file, then update the // sequence file in the database. SequenceFile sf = constructSequenceFile(); - SequenceFile sfUpdated = new LocalSequenceFile(); + SequenceFile sfUpdated = new SequenceFile(); sfUpdated.setFile(sf.getFile()); final Long id = 1L; sf.setId(id); @@ -185,7 +170,7 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { } private SequenceFile constructSequenceFile() throws IOException { - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); Path sequenceFile = Files.createTempFile(null, null); Files.write(sequenceFile, FILE_CONTENTS.getBytes()); sf.setFile(sequenceFile); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index f23d23cd48b..8f5350f9315 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -18,14 +18,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; + import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepositoryImpl; import ca.corefacility.bioinformatics.irida.util.RecursiveDeleteVisitor; @@ -34,9 +27,6 @@ * Tests for {@link FilesystemSupplementedRepositoryImpl}. * */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequenceFileRepositoryImplTest { private static final String TEMP_FILE_PREFIX = UUID.randomUUID().toString().replaceAll("-", ""); @@ -44,14 +34,11 @@ public class SequenceFileRepositoryImplTest { private Path baseDirectory; private EntityManager entityManager; - @Autowired - private IridaFileStorageService iridaFileStorageService; - @Before public void setUp() throws IOException { baseDirectory = Files.createTempDirectory(TEMP_FILE_PREFIX); entityManager = mock(EntityManager.class); - repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageService); + repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory); } @After @@ -65,7 +52,7 @@ private Path getTempFile() throws IOException { @Test public void testCreateFileMissingIdentifier() throws IOException { - SequenceFile s = new LocalSequenceFile(getTempFile()); + SequenceFile s = new SequenceFile(getTempFile()); try { repository.save(s); fail(); @@ -80,7 +67,7 @@ public void testCreateFile() throws IOException { Long lid = new Long(1111); Path f = getTempFile(); String filename = f.getFileName().toString(); - SequenceFile s = new LocalSequenceFile(f); + SequenceFile s = new SequenceFile(f); s.setId(lid); when(entityManager.find(SequenceFile.class, lid)).thenReturn(s); when(entityManager.merge(s)).thenReturn(s); @@ -97,7 +84,7 @@ public void testCreateFile() throws IOException { @Test public void testUpdateFileMissingIdentifier() throws IOException { - SequenceFile s = new LocalSequenceFile(getTempFile()); + SequenceFile s = new SequenceFile(getTempFile()); try { repository.save(s); fail(); @@ -110,7 +97,7 @@ public void testUpdateFileMissingIdentifier() throws IOException { @Test public void testUpdateMissingDirectory() throws IOException { Path f = getTempFile(); - SequenceFile s = new LocalSequenceFile(f); + SequenceFile s = new SequenceFile(f); try { repository.save(s); @@ -128,7 +115,7 @@ public void testUpdateExistingFilename() throws IOException { Long lid = new Long(1111); Path oldFile = getTempFile(); Files.write(oldFile, originalText.getBytes()); - SequenceFile sf = new LocalSequenceFile(oldFile); + SequenceFile sf = new SequenceFile(oldFile); sf.setId(lid); // create the directory and put the file into it. // so call create instead of rewriting the logic: @@ -171,7 +158,7 @@ public void testUpdateExistingFilename() throws IOException { public void testUpdate() throws IOException { Long lId = new Long(9999); Path originalFile = getTempFile(); - SequenceFile original = new LocalSequenceFile(originalFile); + SequenceFile original = new SequenceFile(originalFile); original.setId(lId); when(entityManager.find(SequenceFile.class, lId)).thenReturn(original); when(entityManager.merge(original)).thenReturn(original); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java index f216079999b..078e4be6d8a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/TestDataFactory.java @@ -15,7 +15,6 @@ import java.util.function.Function; import org.springframework.beans.DirectFieldAccessor; -import org.springframework.core.convert.converter.Converter; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; @@ -33,7 +32,6 @@ import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -80,7 +78,7 @@ public static Sample constructSample() { public static SingleEndSequenceFile constructSingleEndSequenceFile(){ Path path = Paths.get("/tmp/sequence-files/fake-file1.fast"); - return new SingleEndSequenceFile(new LocalSequenceFile(path)); + return new SingleEndSequenceFile(new SequenceFile(path)); } /** @@ -146,7 +144,7 @@ public static List generateSequencingObjectsForSampl List join = new ArrayList<>(); for (long i = 0; i < 5; i++) { Path path = Paths.get("/tmp/sequence-files/fake-file" + Math.random() + ".fast"); - SequenceFile file = new LocalSequenceFile(path); + SequenceFile file = new SequenceFile(path); file.setId(i); SingleEndSequenceFile obj = new SingleEndSequenceFile(file); obj.setId(i); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java index b0198f03f89..6cc573be08e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java @@ -1,7 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.unit.web; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.SequencingRunController; @@ -58,7 +58,7 @@ public void testGetFilesPage() throws IOException { ExtendedModelMap model = new ExtendedModelMap(); SequencingRun sequencingRunEntity = new SequencingRun(SequencingRun.LayoutType.PAIRED_END, "miseq"); - ImmutableSet files = ImmutableSet.of(new SingleEndSequenceFile(new LocalSequenceFile())); + ImmutableSet files = ImmutableSet.of(new SingleEndSequenceFile(new SequenceFile())); when(sequencingRunService.read(runId)).thenReturn(sequencingRunEntity); when(objectService.getSequencingObjectsForSequencingRun(sequencingRunEntity)).thenReturn(files); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index d5ece606df7..3b844b36398 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -20,7 +20,6 @@ import org.springframework.ui.ExtendedModelMap; import org.springframework.ui.Model; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; @@ -52,7 +51,7 @@ public void setUp() { controller = new SequenceFileController(objectService, sequencingRunService, analysisService); Path path = Paths.get(FILE_PATH); - SequenceFile file = new LocalSequenceFile(path); + SequenceFile file = new SequenceFile(path); file.setId(FILE_ID); SingleEndSequenceFile seqObject = new SingleEndSequenceFile(file); when(objectService.read(anyLong())).thenReturn(seqObject); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java index 390b722131a..20552d850b7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/projects/ProjectSamplesControllerTest.java @@ -32,7 +32,6 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.Role; @@ -328,7 +327,7 @@ public void testDownloadSamples() throws IOException { MockHttpServletResponse response = new MockHttpServletResponse(); Path path = Paths.get(FILE_PATH); - SequenceFile file = new LocalSequenceFile(path); + SequenceFile file = new SequenceFile(path); ImmutableList filejoin = ImmutableList.of(new SampleSequencingObjectJoin(sample, new SingleEndSequenceFile(file))); @@ -362,7 +361,7 @@ public void testDownloadSamplesWithSameName() throws IOException { MockHttpServletResponse response = new MockHttpServletResponse(); Path path = Paths.get(FILE_PATH); - SequenceFile file = new LocalSequenceFile(path); + SequenceFile file = new SequenceFile(path); ImmutableList filejoin = ImmutableList.of(new SampleSequencingObjectJoin(sample, new SingleEndSequenceFile(file)), new SampleSequencingObjectJoin(sample, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index ace2ef143bc..c19b0144b4e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -25,7 +25,10 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -134,7 +137,7 @@ public void testGetSampleFiles() throws IOException { ExtendedModelMap model = new ExtendedModelMap(); Long sampleId = 1L; Sample sample = new Sample(); - SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); + SequenceFile file = new SequenceFile(Paths.get("/tmp")); file.setId(2L); Project project = new Project(); @@ -163,7 +166,7 @@ public void testGetSampleFilesAsAdmin() throws IOException { ExtendedModelMap model = new ExtendedModelMap(); Long sampleId = 1L; Sample sample = new Sample(); - SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); + SequenceFile file = new SequenceFile(Paths.get("/tmp")); file.setId(2L); List files = Lists.newArrayList(new SampleSequencingObjectJoin(sample, @@ -193,7 +196,7 @@ public void testGetSampleFilesNoAccess() throws IOException { Long sampleId = 1L; Sample sample = new Sample(); - SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); + SequenceFile file = new SequenceFile(Paths.get("/tmp")); file.setId(2L); Project project = new Project(); @@ -224,7 +227,7 @@ public void testRemoveFileFromSample() { Long sampleId = 1L; Long fileId = 2L; Sample sample = new Sample(); - SequencingObject file = new SingleEndSequenceFile(new LocalSequenceFile(Paths.get("/tmp"))); + SequencingObject file = new SingleEndSequenceFile(new SequenceFile(Paths.get("/tmp"))); when(sampleService.read(sampleId)).thenReturn(sample); when(sequencingObjectService.readSequencingObjectForSample(sample, fileId)).thenReturn(file); @@ -242,7 +245,7 @@ public void testDownloadAssembly() throws IOException { Long sampleId = 1L; Long assemblyId = 3L; - SequenceFile file = new LocalSequenceFile(Paths.get("/tmp")); + SequenceFile file = new SequenceFile(Paths.get("/tmp")); file.setId(2L); Sample sample = new Sample(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java index 80b9a97ec83..34aa85af7ca 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/security/permissions/analysis/ReadAnalysisSubmissionPermissionTest.java @@ -20,7 +20,10 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -29,7 +32,6 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.ProjectAnalysisSubmissionJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.repositories.user.UserRepository; -import ca.corefacility.bioinformatics.irida.security.permissions.analysis.ReadAnalysisSubmissionPermission; import ca.corefacility.bioinformatics.irida.security.permissions.files.ReadSequencingObjectPermission; import ca.corefacility.bioinformatics.irida.security.permissions.project.ReadProjectPermission; @@ -82,7 +84,7 @@ public void setUp() { readAnalysisSubmissionPermission = new ReadAnalysisSubmissionPermission(analysisSubmissionRepository, userRepository, sequencingObjectRepository, seqObjectPermission, pasRepository, readProjectPermission); - inputSingleFiles = Sets.newHashSet(new SingleEndSequenceFile(new LocalSequenceFile())); + inputSingleFiles = Sets.newHashSet(new SingleEndSequenceFile(new SequenceFile())); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java index 8a9873b4986..3d372078ae2 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/DatabaseSetupGalaxyITService.java @@ -19,14 +19,15 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; - import com.google.common.collect.Sets; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowState; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -465,7 +466,7 @@ public List setupSequencingObjectInDatabase(Long sampleId List returnedSequenceFiles = new ArrayList<>(); for (Path sequenceFilePath : sequenceFilePaths) { - SingleEndSequenceFile singleEndSequenceFile = new SingleEndSequenceFile(new LocalSequenceFile(sequenceFilePath)); + SingleEndSequenceFile singleEndSequenceFile = new SingleEndSequenceFile(new SequenceFile(sequenceFilePath)); sequencingObjectService.createSequencingObjectInSample(singleEndSequenceFile, sample); waitForFilesToSettle(singleEndSequenceFile); @@ -495,8 +496,8 @@ public List setupSampleSequenceFileInDatabase(long sampleId, L Sample sample = sampleService.read(sampleId); List returnedSequenceFilePairs = new ArrayList<>(); for (int i = 0; i < sequenceFiles1.size(); i++) { - SequenceFile sf1 = new LocalSequenceFile(sequenceFiles1.get(i)); - SequenceFile sf2 = new LocalSequenceFile(sequenceFiles2.get(i)); + SequenceFile sf1 = new SequenceFile(sequenceFiles1.get(i)); + SequenceFile sf2 = new SequenceFile(sequenceFiles2.get(i)); SequenceFilePair pair = new SequenceFilePair(sf1, sf2); sequencingObjectService.createSequencingObjectInSample(pair, sample); waitForFilesToSettle(pair); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java index 1ed4edc6c9c..2bca7cb7d5a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/execution/galaxy/impl/unit/AnalysisExecutionServiceGalaxyTest.java @@ -33,7 +33,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.galaxy.WorkflowUploadException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; @@ -130,7 +130,7 @@ public void setup() throws IridaWorkflowNotFoundException, IOException, Executio MockitoAnnotations.initMocks(this); String submissionName = "name"; - Set submissionInputFiles = Sets.newHashSet(new SingleEndSequenceFile(new LocalSequenceFile())); + Set submissionInputFiles = Sets.newHashSet(new SingleEndSequenceFile(new SequenceFile())); analysisSubmission = AnalysisSubmission.builder(WORKFLOW_ID).name(submissionName + "intial").inputFiles(submissionInputFiles).build(); analysisPreparing = AnalysisSubmission.builder(WORKFLOW_ID).name(submissionName + "preparing").inputFiles(submissionInputFiles).build(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java index 7e8bbf314e6..5ccd78dac03 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java @@ -66,7 +66,10 @@ import ca.corefacility.bioinformatics.irida.exceptions.WorkflowException; import ca.corefacility.bioinformatics.irida.exceptions.galaxy.GalaxyDatasetNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; @@ -218,7 +221,7 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc Files.delete(referenceFilePath); Files.copy(referenceFilePathReal, referenceFilePath); - singleFileSet = Sets.newHashSet(new SingleEndSequenceFile(new LocalSequenceFile(sequenceFilePathA))); + singleFileSet = Sets.newHashSet(new SingleEndSequenceFile(new SequenceFile(sequenceFilePathA))); GalaxyInstance galaxyInstanceAdmin = localGalaxy.getGalaxyInstanceAdmin(); HistoriesClient historiesClient = galaxyInstanceAdmin.getHistoriesClient(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index 3e0a8187312..fb63f03ea5c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -5,7 +5,10 @@ import ca.corefacility.bioinformatics.irida.exceptions.galaxy.GalaxyDatasetException; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.upload.galaxy.GalaxyProjectName; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflowTestBuilder; @@ -151,9 +154,9 @@ public class AnalysisWorkspaceServiceGalaxyTest { public void setup() throws IOException, UploadException, GalaxyDatasetException { MockitoAnnotations.initMocks(this); - sFileA = new LocalSequenceFile(createTempFile("fileA", "fastq")); - sFileB = new LocalSequenceFile(createTempFile("fileB", "fastq")); - sFileC = new LocalSequenceFile(createTempFile("fileC", "fastq")); + sFileA = new SequenceFile(createTempFile("fileA", "fastq")); + sFileB = new SequenceFile(createTempFile("fileB", "fastq")); + sFileC = new SequenceFile(createTempFile("fileC", "fastq")); sObjA = new SingleEndSequenceFile(sFileA); sObjB = new SingleEndSequenceFile(sFileB); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java index 4802315b328..3f032eb2b54 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/ExportUploadServiceTest.java @@ -8,7 +8,6 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.service.impl.TestEmailController; @@ -365,7 +364,7 @@ private NcbiExportSubmission createFakeSubmission(String sequenceFileExtension) NcbiBioSampleFiles ncbiBioSampleFiles = new NcbiBioSampleFiles(); Path tempFile = Files.createTempFile("sequencefile", sequenceFileExtension); - SequenceFile sequenceFile = new LocalSequenceFile(tempFile); + SequenceFile sequenceFile = new SequenceFile(tempFile); sequenceFile.setId(1L); SingleEndSequenceFile singleFile = new SingleEndSequenceFile(sequenceFile); singleFile.setId(1L); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java index 999920cad1e..ab975040874 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/export/NcbiExportSubmissionServiceTest.java @@ -2,7 +2,6 @@ import ca.corefacility.bioinformatics.irida.model.NcbiExportSubmission; import ca.corefacility.bioinformatics.irida.model.export.NcbiBioSampleFiles; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -38,7 +37,7 @@ public void setup() { @Test public void testCreate() { - SingleEndSequenceFile sequenceFile = new SingleEndSequenceFile(new LocalSequenceFile()); + SingleEndSequenceFile sequenceFile = new SingleEndSequenceFile(new SequenceFile()); NcbiBioSampleFiles ncbiBioSampleFiles = new NcbiBioSampleFiles("sample", Sets.newHashSet(sequenceFile), Sets.newHashSet(), null, "library_name", null, null, null, "library_construction_protocol", @@ -53,7 +52,7 @@ public void testCreate() { @Test public void testCreatePairs() { - SequenceFile sequenceFile = new LocalSequenceFile(); + SequenceFile sequenceFile = new SequenceFile(); NcbiBioSampleFiles ncbiBioSampleFiles = new NcbiBioSampleFiles("sample", Sets.newHashSet(), Sets.newHashSet(new SequenceFilePair(sequenceFile, sequenceFile)), null, "library_name", null, null, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java index c80eb498b33..93f0563d1be 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingObjectServiceImplIT.java @@ -60,12 +60,19 @@ import ca.corefacility.bioinformatics.irida.model.sample.QCEntry.QCEntryStatus; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; -import ca.corefacility.bioinformatics.irida.service.*; +import ca.corefacility.bioinformatics.irida.service.AnalysisService; +import ca.corefacility.bioinformatics.irida.service.ProjectService; +import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; +import ca.corefacility.bioinformatics.irida.service.SequencingRunService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @RunWith(SpringJUnit4ClassRunner.class) @@ -168,7 +175,7 @@ public void testGetSequenceFilePairForSample() { @WithMockUser(username = "fbristow", roles = "SEQUENCER") public void testCreateSequenceFileInSample() throws IOException, InterruptedException { Sample s = sampleService.read(1L); - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = new GZIPOutputStream(Files.newOutputStream(sequenceFile)); gzOut.write(FASTQ_FILE_CONTENTS); @@ -200,7 +207,7 @@ public void testCreateSequenceFileInSample() throws IOException, InterruptedExce @WithMockUser(username = "fbristow", roles = "SEQUENCER") public void testCreateCorruptSequenceFileInSample() throws IOException, InterruptedException { Sample s = sampleService.read(1L); - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = Files.newOutputStream(sequenceFile); gzOut.write("not a file".getBytes()); @@ -324,7 +331,7 @@ public void testCreateNotCompressedSequenceFile() throws IOException, Interrupte @WithMockUser(username = "fbristow", roles = "SEQUENCER") public void testCreateCompressedSequenceFile() throws IOException, InterruptedException { final Long expectedRevisionNumber = 2L; - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = new GZIPOutputStream(Files.newOutputStream(sequenceFile)); gzOut.write(FASTQ_FILE_CONTENTS); @@ -422,7 +429,7 @@ public void testCoverage() throws IOException, InterruptedException { project = projectService.update(project); Sample s = sampleService.read(1L); - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); Path sequenceFile = Files.createTempFile("TEMPORARY-SEQUENCE-FILE", ".gz"); OutputStream gzOut = new GZIPOutputStream(Files.newOutputStream(sequenceFile)); gzOut.write(FASTQ_FILE_CONTENTS); @@ -565,6 +572,6 @@ private SequenceFile createSequenceFile(String name) throws IOException{ Path sequenceFile = Files.createTempFile(name, ".fastq"); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - return new LocalSequenceFile(sequenceFile); + return new SequenceFile(sequenceFile); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java index b72f5a24b92..d9074ac39a2 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/SequencingRunServiceImplIT.java @@ -6,7 +6,6 @@ import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun.LayoutType; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -77,7 +76,6 @@ public class SequencingRunServiceImplIT { @Autowired private AnalysisService analysisService; - @Test @WithMockUser(username = "fbristow", password = "password1", roles = "SEQUENCER") public void testAddSequenceFileToMiseqRunAsSequencer() throws IOException, InterruptedException { @@ -112,7 +110,7 @@ private void testAddSequenceFileToMiseqRun() throws IOException, InterruptedExce // that we can link to. Path sequenceFile = Files.createTempFile(null, null); Files.write(sequenceFile, FASTQ_FILE_CONTENTS); - SequenceFile sf = new LocalSequenceFile(sequenceFile); + SequenceFile sf = new SequenceFile(sequenceFile); SequencingObject so = new SingleEndSequenceFile(sf); so = objectService.create(so); @@ -304,7 +302,7 @@ public void testAddDetachedRunToSequenceFile() throws IOException, InterruptedEx Path p = Files.createTempFile(null, null); Files.write(p, FASTQ_FILE_CONTENTS); - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); sf.setFile(p); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); Sample sample = sampleService.read(1L); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java index 714213f49fb..36dce361909 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/sample/SampleServiceImplTest.java @@ -29,7 +29,6 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -104,7 +103,7 @@ public void testGetSampleForProject() { public void testRemoveSequenceFileFromSample() { Sample s = new Sample(); s.setId(1111L); - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); sf.setId(2222L); SingleEndSequenceFile obj = new SingleEndSequenceFile(sf); obj.setId(2L); @@ -249,7 +248,7 @@ public void testGetCoverageForSampleSuccess() throws SequenceFileAnalysisExcepti Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new LocalSequenceFile(); + SequenceFile sf1 = new SequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -305,7 +304,7 @@ public void testGetTotalBasesForSampleSuccessOne() throws SequenceFileAnalysisEx Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new LocalSequenceFile(); + SequenceFile sf1 = new SequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -334,9 +333,9 @@ public void testGetTotalBasesForSampleSuccessTwo() throws SequenceFileAnalysisEx Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new LocalSequenceFile(); + SequenceFile sf1 = new SequenceFile(); sf1.setId(2222L); - SequenceFile sf2 = new LocalSequenceFile(); + SequenceFile sf2 = new SequenceFile(); sf1.setId(3333L); SampleSequencingObjectJoin join1 = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -369,7 +368,7 @@ public void testGetTotalBasesForSampleFailNoFastQC() throws SequenceFileAnalysis Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new LocalSequenceFile(); + SequenceFile sf1 = new SequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -390,7 +389,7 @@ public void testGetTotalBasesForSampleFailMultipleFastQC() throws SequenceFileAn Sample s1 = new Sample(); s1.setId(1L); - SequenceFile sf1 = new LocalSequenceFile(); + SequenceFile sf1 = new SequenceFile(); sf1.setId(2222L); SampleSequencingObjectJoin join = new SampleSequencingObjectJoin(s1, new SingleEndSequenceFile(sf1)); @@ -407,7 +406,7 @@ private Sample s(Long id) { } private SequenceFile sf(Long id) { - SequenceFile sf = new LocalSequenceFile(); + SequenceFile sf = new SequenceFile(); sf.setId(id); try { sf.setFile(Files.createTempFile(null, null)); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java index fc6ee306da4..e89c0487f7b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/remote/impl/SingleEndSequenceFileRemoteServiceImplTest.java @@ -16,7 +16,7 @@ import ca.corefacility.bioinformatics.irida.model.RemoteAPI; import ca.corefacility.bioinformatics.irida.model.remote.RemoteStatus; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.repositories.RemoteAPIRepository; import ca.corefacility.bioinformatics.irida.repositories.remote.SequenceFileRemoteRepository; @@ -45,7 +45,7 @@ public void testGetSequenceFilesForSample() { sample.setRemoteStatus(new RemoteStatus("http://nowhere", api)); - List filesList = Lists.newArrayList(new SingleEndSequenceFile(new LocalSequenceFile())); + List filesList = Lists.newArrayList(new SingleEndSequenceFile(new SequenceFile())); when(apiRepo.getRemoteAPIForUrl(seqFilesHref)).thenReturn(api); when(repository.list(seqFilesHref, api)).thenReturn(filesList); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java index 618b578ca0b..1cfd02008ae 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/TestDataFactory.java @@ -6,7 +6,6 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; @@ -14,57 +13,55 @@ /** * Generates test data for unit tests. - * */ public final class TestDataFactory { /** - * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.User}. - * - * @return a {@link ca.corefacility.bioinformatics.irida.model.User} with identifier. - */ - public static User constructUser() { - User u = new User(); - String username = "fbristow"; - u.setId(1L); - u.setUsername(username); + * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.User}. + * + * @return a {@link ca.corefacility.bioinformatics.irida.model.User} with identifier. + */ + public static User constructUser() { + User u = new User(); + String username = "fbristow"; + u.setId(1L); + u.setUsername(username); - return u; - } + return u; + } - /** - * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.sample.Sample}. - * - * @return a sample with a name and identifier. - */ - public static Sample constructSample() { - String sampleName = "sampleName"; - Sample s = new Sample(); - s.setSampleName(sampleName); - s.setId(1L); - return s; - } + /** + * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.sample.Sample}. + * + * @return a sample with a name and identifier. + */ + public static Sample constructSample() { + String sampleName = "sampleName"; + Sample s = new Sample(); + s.setSampleName(sampleName); + s.setId(1L); + return s; + } + + /** + * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile}. + * + * @return a {@link ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile} with identifier. + */ + public static SequenceFile constructSequenceFile() throws IOException { + Path f = Files.createTempFile(null, null); + SequenceFile sf = new SequenceFile(); + sf.setId(1L); + sf.setFile(f); + return sf; + } - /** - * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile}. - * - * @return a {@link ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile} with identifier. - */ - public static SequenceFile constructSequenceFile() throws IOException { - Path f = Files.createTempFile(null, null); - SequenceFile sf = new LocalSequenceFile(); - sf.setId(1L); - sf.setFile(f); - return sf; - } - /** * Construct a simple {@link SingleEndSequenceFile} - * + * * @return a {@link SingleEndSequenceFile} with a {@link SequenceFile} and - * id - * @throws IOException - * if the temp file couldn't be created + * id + * @throws IOException if the temp file couldn't be created */ public static SingleEndSequenceFile constructSingleEndSequenceFile() throws IOException { SequenceFile sf = constructSequenceFile(); @@ -72,13 +69,12 @@ public static SingleEndSequenceFile constructSingleEndSequenceFile() throws IOEx sesf.setId(2L); return sesf; } - + /** * Construct a simple {@link SequenceFilePair} object - * + * * @return a {@link SequenceFilePair} - * @throws IOException - * if the temp files couldn't be created + * @throws IOException if the temp files couldn't be created */ public static SequenceFilePair constructSequenceFilePair() throws IOException { SequenceFile sf1 = constructSequenceFile(); @@ -90,14 +86,14 @@ public static SequenceFilePair constructSequenceFilePair() throws IOException { return pair; } - /** - * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.Project}. - * - * @return a project with a name and identifier. - */ - public static Project constructProject() { - Project p = new Project(); - p.setId(1L); - return p; - } + /** + * Construct a simple {@link ca.corefacility.bioinformatics.irida.model.Project}. + * + * @return a project with a name and identifier. + */ + public static Project constructProject() { + Project p = new Project(); + p.setId(1L); + return p; + } } From 776546d8ba439f413b89f2f9b9679a1f2f9859bc Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 17:26:46 -0500 Subject: [PATCH 034/655] Removed newlines --- .../concatenate/impl/SequenceFilePairConcatenatorTest.java | 5 ----- .../impl/SingleEndSequenceFileConcatenatorTest.java | 1 - .../irida/processing/impl/unit/FastqcFileProcessorTest.java | 3 --- .../irida/ria/unit/web/SequencingRunControllerTest.java | 1 - .../irida/ria/unit/web/samples/SamplesControllerTest.java | 1 - 5 files changed, 11 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index a15a25a024a..06254aac387 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -6,11 +6,6 @@ import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 9401f97ffa0..d40f09f63cd 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -3,7 +3,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; - import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index a244420e8f9..0db5ebd1086 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -34,8 +34,6 @@ import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -import com.github.springtestdbunit.DbUnitTestExecutionListener; - /** * Tests for {@link FastqcFileProcessor}. * @@ -46,7 +44,6 @@ public class FastqcFileProcessorTest { private SequenceFileRepository sequenceFileRepository; private AnalysisOutputFileRepository outputFileRepository; private MessageSource messageSource; - private static final Logger logger = LoggerFactory.getLogger(FastqcFileProcessorTest.class); private static final String SEQUENCE = "ACGTACGTN"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java index 6cc573be08e..dff3d75c572 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java @@ -24,7 +24,6 @@ public class SequencingRunControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; - @Before public void setup() { sequencingRunService = mock(SequencingRunService.class); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index c19b0144b4e..349f7149b3a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -62,7 +62,6 @@ public class SamplesControllerTest { private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; - @Before public void setUp() { sampleService = mock(SampleService.class); From fb554ab4139a3c322d7e6c0c362a2276c8958658 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 18:29:22 -0500 Subject: [PATCH 035/655] Updated changeset for sequence-file dtype --- .../changesets/20.05/add-dtype-to-sequence-file.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml index 16ff7998fff..af1c9b215d4 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml @@ -7,13 +7,16 @@ - - - + + UPDATE sequence_file SET DTYPE="LocalSequenceFile" + + + UPDATE sequence_file_AUD SET DTYPE="LocalSequenceFile" + From b13db5862b22ca2e286103b78c67bb7dd76d8f1d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Apr 2020 19:47:13 -0500 Subject: [PATCH 036/655] Added default value for DTYPE --- .../database/changesets/20.05/add-dtype-to-sequence-file.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml index af1c9b215d4..38102420b0b 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml @@ -7,7 +7,7 @@ - + UPDATE sequence_file SET DTYPE="LocalSequenceFile" From efc8cfb005541a735b06c86670e7ff2d6f6aebac Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 21 Apr 2020 12:36:37 -0500 Subject: [PATCH 037/655] Updated changeset to set a default value for dtype if one isn't provided --- .../changesets/20.05/add-dtype-to-sequence-file.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml index 38102420b0b..f5202f65817 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml @@ -7,7 +7,9 @@ - + + + UPDATE sequence_file SET DTYPE="LocalSequenceFile" @@ -16,7 +18,7 @@ - UPDATE sequence_file_AUD SET DTYPE="LocalSequenceFile" + UPDATE sequence_file_aud SET DTYPE="LocalSequenceFile" From 4f86e81e46cf9e77434f4acf04a4173e6c11ed28 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 21 Apr 2020 19:19:59 -0500 Subject: [PATCH 038/655] Removed local and cloud sequence files as we can access the necessary data using the iridafilestorageservice in sequence file using @postload --- .../irida/model/assembly/GenomeAssembly.java | 11 ++++ .../model/sequenceFile/CloudSequenceFile.java | 56 ------------------- .../model/sequenceFile/LocalSequenceFile.java | 53 ------------------ .../model/sequenceFile/SequenceFile.java | 18 ++++-- .../impl/SequenceFilePairConcatenator.java | 4 +- .../SingleEndSequenceFileConcatenator.java | 2 +- .../IridaFileStorageLocalServiceImpl.java | 17 ------ .../filesystem/IridaFileStorageService.java | 21 ------- .../web/samples/SamplesAjaxController.java | 7 +-- .../sequencefile/SequenceFileResource.java | 1 + .../RESTSampleSequenceFilesController.java | 7 +-- .../20.05/add-dtype-to-sequence-file.xml | 24 -------- .../database/changesets/20.05/all-changes.xml | 3 - .../samples/SamplesAjaxControllerTest.java | 7 +-- .../SampleSequenceFilesControllerTest.java | 22 ++++---- 15 files changed, 44 insertions(+), 209 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java delete mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 66608db4591..aa4b97eee7d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -80,6 +80,17 @@ public Date getCreatedDate() { return createdDate; } + /** + * Get the size of the genome assembly files + * + * @return file size + * @throws IOException if the file cannot be read + */ + @JsonIgnore + public long getFileSizeBytes() throws IOException { + return Files.size(getFile()); + } + /** * Add a sample to this assembly * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java deleted file mode 100644 index 8b3de03e8c8..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/CloudSequenceFile.java +++ /dev/null @@ -1,56 +0,0 @@ -package ca.corefacility.bioinformatics.irida.model.sequenceFile; - -import java.nio.file.Path; - -import javax.persistence.DiscriminatorValue; -import javax.persistence.Entity; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import ca.corefacility.bioinformatics.irida.model.IridaThing; -import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sample.Sample; - -import com.fasterxml.jackson.annotation.JsonIgnore; - -/** - * A file that may be stored somewhere on a cloud file system and belongs to a - * particular {@link Sample}. - */ -@Entity -public class CloudSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { - private static final Logger logger = LoggerFactory.getLogger(CloudSequenceFile.class); - - - public CloudSequenceFile() { - super(); - } - - public CloudSequenceFile(Path sampleFile) { - super(sampleFile); - } - - @Override - public String getLabel() { - // Since the filesystem is virtual the path and filename are just one string - // so we split on the "/" and take the last token which is the file name - String [] cloudFileNameTokens = super.getFile().toString().split("/"); - return cloudFileNameTokens[cloudFileNameTokens.length-1]; - } - /** - * Get the size of the file. - * - * @return The String representation of the file size - */ - @JsonIgnore - @Override - public String getFileSize() { - String size = "N/A"; - super.getIridaFileStorageService().getFileSize(super.getFile()); - // Need to implement cloud code to get file size - size = IridaSequenceFile.humanReadableByteCount(super.getIridaFileStorageService().getFileSize(super.getFile()), true); - return size; - } - -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java deleted file mode 100644 index c22a35fab7f..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/LocalSequenceFile.java +++ /dev/null @@ -1,53 +0,0 @@ -package ca.corefacility.bioinformatics.irida.model.sequenceFile; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; - -import javax.persistence.DiscriminatorValue; -import javax.persistence.Entity; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import ca.corefacility.bioinformatics.irida.model.IridaThing; -import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; -import ca.corefacility.bioinformatics.irida.model.sample.Sample; - -import com.fasterxml.jackson.annotation.JsonIgnore; - -/** - * A file that may be stored somewhere on the local file system and belongs to a - * particular {@link Sample}. - */ -@Entity -public class LocalSequenceFile extends SequenceFile implements IridaSequenceFile, IridaThing { - private static final Logger logger = LoggerFactory.getLogger(LocalSequenceFile.class); - - public LocalSequenceFile() { - super(); - } - - public LocalSequenceFile(Path sampleFile) { - super(sampleFile); - } - - @Override - public String getLabel() { - return super.getFile().getFileName().toString(); - } - /** - * Get the size of the file. - * - * @return The String representation of the file size - */ - @JsonIgnore - @Override - public String getFileSize() { - String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(super.getIridaFileStorageService().getFileSize(super.getFile()), true); - return size; - } - -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 0f7712d5f12..4f58161be90 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -40,11 +40,10 @@ * particular {@link Sample}. */ @Entity -@Inheritance(strategy=InheritanceType.SINGLE_TABLE) @Table(name = "sequence_file") @Audited @EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, SequenceFileListener.class }) -public abstract class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, +public class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, VersionedFileFields, IridaSequenceFile, RemoteSynchronizable { private static final Logger logger = LoggerFactory.getLogger(SequenceFile.class); @@ -57,7 +56,7 @@ public abstract class SequenceFile extends IridaResourceSupport implements Mutab @NotNull(message = "{sequencefile.file.notnull}") @Column(name = "file_path", unique = true) - protected Path file; + private Path file; @CreatedDate @NotNull @@ -108,14 +107,23 @@ public SequenceFile() { * * @return the file label. */ - public abstract String getLabel(); + @Override + public String getLabel() { + return file.getFileName().toString(); + } /** * Get the implementation-specific file size. * * @return the file size. */ - public abstract String getFileSize(); + @Override + @JsonIgnore + public String getFileSize() { + String size = "N/A"; + size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageService.getFileSize(getFile()), true); + return size; + } /** * Create a new {@link SequenceFile} with the given file Path diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 094be469685..043b9896186 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -70,8 +70,8 @@ public SequenceFilePair concatenateFiles(List toConc } // create new SequenceFiles - SequenceFile forward = iridaFileStorageService.createSequenceFile(forwardFile); - SequenceFile reverse = iridaFileStorageService.createSequenceFile(reverseFile); + SequenceFile forward = new SequenceFile(forwardFile); + SequenceFile reverse = new SequenceFile(reverseFile); // create the new pair SequenceFilePair sequenceFilePair = new SequenceFilePair(forward, reverse); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 0aee792cf16..debfd2a54a1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -62,7 +62,7 @@ public SingleEndSequenceFile concatenateFiles(List t } // create the new sequencefile and object - SequenceFile forward = iridaFileStorageService.createSequenceFile(tempFile); + SequenceFile forward = new SequenceFile(tempFile); SingleEndSequenceFile seqObject = new SingleEndSequenceFile(forward); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java index 33209cead37..757697c5f9d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalServiceImpl.java @@ -19,7 +19,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.LocalSequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -161,22 +160,6 @@ public boolean isGzipped(Path file) throws IOException { } } - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createEmptySequenceFile() { - return new LocalSequenceFile(); - } - - /** - * {@inheritDoc} - */ - @Override - public SequenceFile createSequenceFile(Path file) { - return new LocalSequenceFile(file); - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index d0a50490c2d..6723d50ed21 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -106,27 +106,6 @@ public interface IridaFileStorageService { */ public boolean isGzipped(Path file) throws IOException; - /** - * Creates an empty SequenceFile depending on - * storage type - * - * @return Local or Cloud {@link SequenceFile} object - * @throws IOException if file can't be read - * - */ - public SequenceFile createEmptySequenceFile(); - - /** - * Creates a SequenceFile depending on - * storage type - * - * @param file The path to the file - * @return Local or Cloud {@link SequenceFile} object - * @throws IOException if file can't be read - * - */ - public SequenceFile createSequenceFile(Path file); - /** * Append a {@link SequenceFile} to a {@link Path} on the filesystem * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index cdc622cb33b..cb02d091cda 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -21,7 +21,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -36,16 +35,14 @@ public class SamplesAjaxController { private final SequencingObjectService sequencingObjectService; private final GenomeAssemblyService genomeAssemblyService; private final MessageSource messageSource; - private IridaFileStorageService iridaFileStorageService; @Autowired public SamplesAjaxController(SampleService sampleService, SequencingObjectService sequencingObjectService, - GenomeAssemblyService genomeAssemblyService, MessageSource messageSource, IridaFileStorageService iridaFileStorageService) { + GenomeAssemblyService genomeAssemblyService, MessageSource messageSource) { this.sampleService = sampleService; this.sequencingObjectService = sequencingObjectService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; - this.iridaFileStorageService = iridaFileStorageService; } /** @@ -162,6 +159,6 @@ private SequenceFile createSequenceFile(MultipartFile file) throws IOException { Path temp = Files.createTempDirectory(null); Path target = temp.resolve(file.getOriginalFilename()); file.transferTo(target.toFile()); - return iridaFileStorageService.createSequenceFile(target); + return new SequenceFile(target); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java index 9376b031282..8d4203a5ee7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java @@ -20,6 +20,7 @@ public class SequenceFileResource { public SequenceFileResource() { + resource = new SequenceFile(); } public SequenceFileResource(SequenceFile sequenceFile) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 08763e8cace..9b18a7db64c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -36,7 +36,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -135,19 +134,17 @@ public class RESTSampleSequenceFilesController { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; - private IridaFileStorageService iridaFileStorageService; protected RESTSampleSequenceFilesController() { } @Autowired public RESTSampleSequenceFilesController(SampleService sampleService, SequencingRunService miseqRunService, - SequencingObjectService sequencingObjectService, AnalysisService analysisService, IridaFileStorageService iridaFileStorageService) { + SequencingObjectService sequencingObjectService, AnalysisService analysisService) { this.sampleService = sampleService; this.miseqRunService = miseqRunService; this.sequencingObjectService = sequencingObjectService; this.analysisService = analysisService; - this.iridaFileStorageService = iridaFileStorageService; } /** @@ -417,7 +414,7 @@ public ModelMap addNewSequenceFileToSample(@PathVariable Long sampleId, @Request logger.trace("Read miseq run " + miseqRunId); } } else { - sf = iridaFileStorageService.createEmptySequenceFile(); + sf = new SequenceFile(); } sf.setFile(target); diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml deleted file mode 100644 index f5202f65817..00000000000 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/add-dtype-to-sequence-file.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - UPDATE sequence_file SET DTYPE="LocalSequenceFile" - - - - - - UPDATE sequence_file_aud SET DTYPE="LocalSequenceFile" - - - diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml index a425dd5005a..76fb311ba46 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/20.05/all-changes.xml @@ -8,7 +8,4 @@ relativeToChangelogFile="true"/> - - diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index bbbbf539c36..112f4a62508 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -10,7 +10,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -25,7 +24,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; @@ -46,9 +44,6 @@ public class SamplesAjaxControllerTest { private SequencingObjectService sequencingObjectService; private GenomeAssemblyService genomeAssemblyService; - @Autowired - private IridaFileStorageService iridaFileStorageService; - /* TEST DATA */ @@ -71,7 +66,7 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource, iridaFileStorageService); + controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); // Set up mocks when(sampleService.read(SAMPLE.getId())).thenReturn(SAMPLE); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index 49f163f7a09..e6ce54d1b49 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -245,7 +245,7 @@ public void testAddNewSequenceFileToSample() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile = new SequenceFile(); SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(6L); @@ -297,7 +297,7 @@ public void testAddNewSequenceFileToSampleCompletedRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile = new SequenceFile(); SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(8L); @@ -323,7 +323,7 @@ public void testAddNewSequenceFileToSampleErrorRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile = new SequenceFile(); SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); resource.setMiseqRunId(8L); Path f = Files.createTempFile(null, null); @@ -360,8 +360,8 @@ public void testAddNewSequenceFilePairToSample() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = new SequenceFile(); + SequenceFile emptySequenceFile2 = new SequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); @@ -437,8 +437,8 @@ public void testAddNewSequenceFilePairToSampleMismatchedRunIDs() throws IOExcept SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = new SequenceFile(); + SequenceFile emptySequenceFile2 = new SequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); @@ -466,8 +466,8 @@ public void testAddNewSequenceFilePairToSampleCompletedRun() throws IOException SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = new SequenceFile(); + SequenceFile emptySequenceFile2 = new SequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); @@ -499,8 +499,8 @@ public void testAddNewSequenceFilePairToSampleErrorRun() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = iridaFileStorageService.createEmptySequenceFile(); - SequenceFile emptySequenceFile2 = iridaFileStorageService.createEmptySequenceFile(); + SequenceFile emptySequenceFile1 = new SequenceFile(); + SequenceFile emptySequenceFile2 = new SequenceFile(); SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); From 0b3307619fcb57db8f0e775614101e9bef09cb76 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 21 Apr 2020 22:33:16 -0500 Subject: [PATCH 039/655] Updated tests --- ...quencingObjectConcatenatorFactoryTest.java | 27 ++++++++++++++----- .../SequenceFilePairConcatenatorTest.java | 16 ++++++++++- ...SingleEndSequenceFileConcatenatorTest.java | 16 ++++++++++- .../impl/unit/ChecksumFileProcessorTest.java | 15 ++++++++++- .../unit/DefaultFileProcessingChainTest.java | 15 ++++++----- .../impl/unit/FastqcFileProcessorTest.java | 15 ++++++++++- .../impl/unit/GzipFileProcessorTest.java | 18 ++++++++++--- .../SequenceFileRepositoryImplTest.java | 15 ++++++++++- .../SampleSequenceFilesControllerTest.java | 6 +---- 9 files changed, 118 insertions(+), 25 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index 96eb4f1893e..52c66de6cb7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -1,12 +1,20 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + import com.google.common.collect.Sets; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.util.Set; @@ -15,43 +23,50 @@ /** * Unit test for {@link SequencingObjectConcatenatorFactory} */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequencingObjectConcatenatorFactoryTest { + + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class); + SingleEndSequenceFile.class, iridaFileStorageService); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class); + SequenceFilePair.class, iridaFileStorageService); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageService); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 06254aac387..8b141d3b197 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -1,11 +1,19 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -14,6 +22,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequenceFilePairConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" @@ -21,9 +32,12 @@ public class SequenceFilePairConcatenatorTest { SequenceFilePairConcatenator concat; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() { - concat = new SequenceFilePairConcatenator(); + concat = new SequenceFilePairConcatenator(iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index d40f09f63cd..ec212b1e8fb 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -1,11 +1,19 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -14,6 +22,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SingleEndSequenceFileConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" @@ -21,9 +32,12 @@ public class SingleEndSequenceFileConcatenatorTest { SingleEndSequenceFileConcatenator concat; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() { - concat = new SingleEndSequenceFileConcatenator(); + concat = new SingleEndSequenceFileConcatenator(iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index ab25ff88a07..b2d5f895fa1 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -11,24 +11,37 @@ import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.ChecksumFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class ChecksumFileProcessorTest { private ChecksumFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; private static final String CHECKSUM = "aeaa0755dc44b393ffe12f02e9bd42b0169b12ca9c15708085db6a4ac9110ee0"; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new ChecksumFileProcessor(sequenceFileRepository); + fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java index 3e76018557d..ba8adc740b0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.DefaultFileProcessingChain; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -38,6 +39,7 @@ public class DefaultFileProcessingChainTest { private SequencingObjectRepository objectRepository; private QCEntryRepository qcRepository; + private IridaFileStorageService iridaFileStorageService; private SequencingObject seqObject; private Long objectId = 1L; @@ -46,6 +48,7 @@ public class DefaultFileProcessingChainTest { public void setUp() { this.objectRepository = mock(SequencingObjectRepository.class); this.qcRepository = mock(QCEntryRepository.class); + this.iridaFileStorageService = mock(IridaFileStorageService.class); seqObject = new NoFileSequencingObject(); when(objectRepository.findById(objectId)).thenReturn(Optional.of(seqObject)); @@ -53,7 +56,7 @@ public void setUp() { @Test(expected = FileProcessorTimeoutException.class) public void testExceedsTimeout() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); fileProcessingChain.setTimeout(1); fileProcessingChain.setSleepDuration(0); @@ -62,7 +65,7 @@ public void testExceedsTimeout() throws FileProcessorTimeoutException { @Test public void testProcessEmptyChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); when(objectRepository.existsById(objectId)).thenReturn(true); fileProcessingChain.launchChain(objectId); @@ -70,7 +73,7 @@ public void testProcessEmptyChain() throws FileProcessorTimeoutException { @Test public void testFailWithContinueChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -84,7 +87,7 @@ public void testFailWithContinueChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFastFailProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -96,7 +99,7 @@ public void testFastFailProcessorChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFailOnProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -106,7 +109,7 @@ public void testFailOnProcessorChain() throws FileProcessorTimeoutException { @Test public void testFailWriteQCEntry() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 0db5ebd1086..59bee091297 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -16,14 +16,20 @@ import java.nio.file.Path; import java.util.Iterator; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.util.ReflectionUtils; import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; @@ -32,6 +38,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -39,6 +46,9 @@ * * */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class FastqcFileProcessorTest { private FastqcFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; @@ -51,12 +61,15 @@ public class FastqcFileProcessorTest { + SEQUENCE + "\n+\n?????????"; private static final String FASTA_FILE_CONTENTS = ">test read\n" + SEQUENCE; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() { messageSource = mock(MessageSource.class); sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); - fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository); + fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageService); } @Test(expected = FileProcessorException.class) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 8edbeea806c..ac1f2d0711e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -16,12 +16,19 @@ import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -29,16 +36,21 @@ * * */ -public class GzipFileProcessorTest { +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it")public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; + @Autowired + private IridaFileStorageService iridaFileStorageService; + @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageService); } @Test(expected = FileProcessorException.class) @@ -60,7 +72,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageService); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index 8f5350f9315..8667ab6b856 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -18,7 +18,13 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepositoryImpl; import ca.corefacility.bioinformatics.irida.util.RecursiveDeleteVisitor; @@ -27,6 +33,9 @@ * Tests for {@link FilesystemSupplementedRepositoryImpl}. * */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = { IridaApiServicesConfig.class }) +@ActiveProfiles("it") public class SequenceFileRepositoryImplTest { private static final String TEMP_FILE_PREFIX = UUID.randomUUID().toString().replaceAll("-", ""); @@ -34,11 +43,15 @@ public class SequenceFileRepositoryImplTest { private Path baseDirectory; private EntityManager entityManager; + @Autowired + private IridaFileStorageService iridaFileStorageService; + + @Before public void setUp() throws IOException { baseDirectory = Files.createTempDirectory(TEMP_FILE_PREFIX); entityManager = mock(EntityManager.class); - repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory); + repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageService); } @After diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index e6ce54d1b49..c5a1766fa42 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -46,7 +46,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -72,9 +71,6 @@ public class SampleSequenceFilesControllerTest { private AnalysisService analysisService; private SequencingRun sequencingRun; - @Autowired - private IridaFileStorageService iridaFileStorageService; - @Before public void setUp() { sampleService = mock(SampleService.class); @@ -83,7 +79,7 @@ public void setUp() { analysisService = mock(AnalysisService.class); sequencingRun = mock(SequencingRun.class); - controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService, iridaFileStorageService); + controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService); } @Test From ac612b8e3c1695f82a346cadae65df756e19c840 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 09:49:52 -0500 Subject: [PATCH 040/655] Reverted changes --- .../SampleSequenceFilesControllerTest.java | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index c5a1766fa42..fc3bd1a8c03 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -241,9 +241,7 @@ public void testAddNewSequenceFileToSample() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = new SequenceFile(); - - SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); + SequenceFileResource resource = new SequenceFileResource(); resource.setMiseqRunId(6L); Path f = Files.createTempFile(null, null); MockMultipartFile mmf = new MockMultipartFile("filename", "filename", "blurgh", FileCopyUtils.copyToByteArray(f @@ -293,9 +291,7 @@ public void testAddNewSequenceFileToSampleCompletedRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = new SequenceFile(); - - SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); + SequenceFileResource resource = new SequenceFileResource(); resource.setMiseqRunId(8L); Path f = Files.createTempFile(null, null); MockMultipartFile mmf = new MockMultipartFile("filename", "filename", "blurgh", FileCopyUtils.copyToByteArray(f @@ -319,8 +315,7 @@ public void testAddNewSequenceFileToSampleErrorRun() throws IOException { SequenceFile sf = so.getSequenceFile(); SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, so); - SequenceFile emptySequenceFile = new SequenceFile(); - SequenceFileResource resource = new SequenceFileResource(emptySequenceFile); + SequenceFileResource resource = new SequenceFileResource(); resource.setMiseqRunId(8L); Path f = Files.createTempFile(null, null); MockMultipartFile mmf = new MockMultipartFile("filename", "filename", "blurgh", FileCopyUtils.copyToByteArray(f @@ -356,11 +351,8 @@ public void testAddNewSequenceFilePairToSample() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = new SequenceFile(); - SequenceFile emptySequenceFile2 = new SequenceFile(); - - SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); - SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); + SequenceFileResource resource1 = new SequenceFileResource(); + SequenceFileResource resource2 = new SequenceFileResource(); resource1.setMiseqRunId(7L); resource2.setMiseqRunId(7L); Path f1 = Files.createTempFile(null, null); @@ -433,11 +425,8 @@ public void testAddNewSequenceFilePairToSampleMismatchedRunIDs() throws IOExcept SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = new SequenceFile(); - SequenceFile emptySequenceFile2 = new SequenceFile(); - - SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); - SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); + SequenceFileResource resource1 = new SequenceFileResource(); + SequenceFileResource resource2 = new SequenceFileResource(); resource1.setMiseqRunId(1L); resource2.setMiseqRunId(2L); Path f1 = Files.createTempFile(null, null); @@ -462,11 +451,8 @@ public void testAddNewSequenceFilePairToSampleCompletedRun() throws IOException SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = new SequenceFile(); - SequenceFile emptySequenceFile2 = new SequenceFile(); - - SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); - SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); + SequenceFileResource resource1 = new SequenceFileResource(); + SequenceFileResource resource2 = new SequenceFileResource(); resource1.setMiseqRunId(4L); resource2.setMiseqRunId(4L); Path f1 = Files.createTempFile(null, null); @@ -495,11 +481,8 @@ public void testAddNewSequenceFilePairToSampleErrorRun() throws IOException { SampleSequencingObjectJoin sso = new SampleSequencingObjectJoin(s, pair); - SequenceFile emptySequenceFile1 = new SequenceFile(); - SequenceFile emptySequenceFile2 = new SequenceFile(); - - SequenceFileResource resource1 = new SequenceFileResource(emptySequenceFile1); - SequenceFileResource resource2 = new SequenceFileResource(emptySequenceFile2); + SequenceFileResource resource1 = new SequenceFileResource(); + SequenceFileResource resource2 = new SequenceFileResource(); resource1.setMiseqRunId(4L); resource2.setMiseqRunId(4L); Path f1 = Files.createTempFile(null, null); From e9fd1c40da2e2c12b48c3c5ff80c93c0ad093ba8 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 09:53:41 -0500 Subject: [PATCH 041/655] Removed unused imports --- .../unit/samples/SampleSequenceFilesControllerTest.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index fc3bd1a8c03..0211b67c24d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -16,21 +16,15 @@ import java.util.Iterator; import java.util.List; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.enums.SequencingRunUploadStatus; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.Matchers; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.hateoas.Link; import org.springframework.http.HttpStatus; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.mock.web.MockMultipartFile; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.ui.ModelMap; import org.springframework.util.FileCopyUtils; @@ -60,9 +54,6 @@ /** * Unit tests for {@link RESTSampleSequenceFilesController}. */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SampleSequenceFilesControllerTest { private RESTSampleSequenceFilesController controller; private SampleService sampleService; From 69e3137d412125192c62b9090fdd9a1ccb3a483e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 09:57:53 -0500 Subject: [PATCH 042/655] Removed unused imports --- .../ria/unit/web/samples/SamplesAjaxControllerTest.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 112f4a62508..9b667c25fad 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -8,18 +8,13 @@ import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.springframework.context.MessageSource; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockMultipartFile; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.web.multipart.MultipartHttpServletRequest; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; @@ -36,9 +31,6 @@ import static org.mockito.Matchers.eq; import static org.mockito.Mockito.*; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SamplesAjaxControllerTest { private SamplesAjaxController controller; private SequencingObjectService sequencingObjectService; From d54618ee31989fe484c4e4d5c9d1b6cc30f5a239 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 10:59:45 -0500 Subject: [PATCH 043/655] Mocked iridafilestorageservice instead of autowiring --- ...quencingObjectConcatenatorFactoryTest.java | 19 +++++++++---------- .../SequenceFilePairConcatenatorTest.java | 16 ++++------------ ...SingleEndSequenceFileConcatenatorTest.java | 16 ++++------------ .../impl/unit/ChecksumFileProcessorTest.java | 13 ++----------- .../unit/DefaultFileProcessingChainTest.java | 3 ++- .../impl/unit/FastqcFileProcessorTest.java | 7 ++----- .../impl/unit/GzipFileProcessorTest.java | 14 +++----------- .../SequenceFileRepositoryImplTest.java | 13 +------------ .../unit/SequencingObjectServiceTest.java | 17 +++-------------- 9 files changed, 30 insertions(+), 88 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index 52c66de6cb7..38e2a3a4517 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -1,36 +1,35 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Sets; + +import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.util.Set; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; /** * Unit test for {@link SequencingObjectConcatenatorFactory} */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequencingObjectConcatenatorFactoryTest { - @Autowired private IridaFileStorageService iridaFileStorageService; + @Before + public void setUp() { + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + } + @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 8b141d3b197..b46d51c1dba 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -1,19 +1,14 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -21,22 +16,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequenceFilePairConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" + SEQUENCE + "\n+\n?????????").getBytes(); - SequenceFilePairConcatenator concat; - - @Autowired + private SequenceFilePairConcatenator concat; private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); concat = new SequenceFilePairConcatenator(iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index ec212b1e8fb..2a4b41336d3 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -1,19 +1,14 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.google.common.collect.Lists; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.IOException; import java.nio.file.Files; @@ -21,22 +16,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SingleEndSequenceFileConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; private static final byte[] FASTQ_FILE_CONTENTS = ("@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" + SEQUENCE + "\n+\n?????????").getBytes(); - SingleEndSequenceFileConcatenator concat; - - @Autowired + private SingleEndSequenceFileConcatenator concat; private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); concat = new SingleEndSequenceFileConcatenator(iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index b2d5f895fa1..1670119d15c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -11,36 +11,27 @@ import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.ChecksumFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class ChecksumFileProcessorTest { private ChecksumFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; private static final String CHECKSUM = "aeaa0755dc44b393ffe12f02e9bd42b0169b12ca9c15708085db6a4ac9110ee0"; - - @Autowired private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java index ba8adc740b0..405c07099af 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java @@ -26,6 +26,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.DefaultFileProcessingChain; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -48,7 +49,7 @@ public class DefaultFileProcessingChainTest { public void setUp() { this.objectRepository = mock(SequencingObjectRepository.class); this.qcRepository = mock(QCEntryRepository.class); - this.iridaFileStorageService = mock(IridaFileStorageService.class); + this.iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); seqObject = new NoFileSequencingObject(); when(objectRepository.findById(objectId)).thenReturn(Optional.of(seqObject)); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 59bee091297..7ee86c5f09c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -38,6 +38,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -46,9 +47,6 @@ * * */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class FastqcFileProcessorTest { private FastqcFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; @@ -60,8 +58,6 @@ public class FastqcFileProcessorTest { private static final String FASTQ_FILE_CONTENTS = "@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" + SEQUENCE + "\n+\n?????????"; private static final String FASTA_FILE_CONTENTS = ">test read\n" + SEQUENCE; - - @Autowired private IridaFileStorageService iridaFileStorageService; @Before @@ -69,6 +65,7 @@ public void setUp() { messageSource = mock(MessageSource.class); sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index ac1f2d0711e..b4768efbabf 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -16,18 +16,13 @@ import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -36,20 +31,17 @@ * * */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it")public class GzipFileProcessorTest { +public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; - - @Autowired private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index 8667ab6b856..4d7d78f8762 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -18,13 +18,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepositoryImpl; import ca.corefacility.bioinformatics.irida.util.RecursiveDeleteVisitor; @@ -33,24 +27,19 @@ * Tests for {@link FilesystemSupplementedRepositoryImpl}. * */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequenceFileRepositoryImplTest { private static final String TEMP_FILE_PREFIX = UUID.randomUUID().toString().replaceAll("-", ""); private FilesystemSupplementedRepositoryImpl repository; private Path baseDirectory; private EntityManager entityManager; - - @Autowired private IridaFileStorageService iridaFileStorageService; - @Before public void setUp() throws IOException { baseDirectory = Files.createTempDirectory(TEMP_FILE_PREFIX); entityManager = mock(EntityManager.class); + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java index f72072517b2..ff84de02946 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java @@ -1,12 +1,12 @@ package ca.corefacility.bioinformatics.irida.service.impl.unit; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun.LayoutType; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; @@ -17,11 +17,6 @@ import ca.corefacility.bioinformatics.irida.web.controller.test.unit.TestDataFactory; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import javax.validation.Validator; import java.io.IOException; @@ -29,9 +24,6 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes = { IridaApiServicesConfig.class }) -@ActiveProfiles("it") public class SequencingObjectServiceTest { SequencingObjectService service; @@ -40,18 +32,15 @@ public class SequencingObjectServiceTest { SampleSequencingObjectJoinRepository ssoRepository; SequenceConcatenationRepository concatenationRepository; Validator validator; - - @Autowired - private IridaFileStorageService iridaFileStorageService; + IridaFileStorageService iridaFileStorageService; @Before public void setUp() { repository = mock(SequencingObjectRepository.class); sequenceFileRepository = mock(SequenceFileRepository.class); ssoRepository = mock(SampleSequencingObjectJoinRepository.class); - concatenationRepository = mock(SequenceConcatenationRepository.class); - + iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); service = new SequencingObjectServiceImpl(repository, sequenceFileRepository, ssoRepository, concatenationRepository, validator, iridaFileStorageService); } From fed5fd3d68ed5cbbe16a18c53300cd501b022c29 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 11:59:52 -0500 Subject: [PATCH 044/655] Removed mocks for iridafilestoragelocalserviceimpl and instantiated instead --- .../impl/SequenceFilePairConcatenatorTest.java | 2 +- .../impl/SingleEndSequenceFileConcatenatorTest.java | 2 +- .../processing/impl/unit/ChecksumFileProcessorTest.java | 2 +- .../processing/impl/unit/FastqcFileProcessorTest.java | 8 +------- .../irida/processing/impl/unit/GzipFileProcessorTest.java | 2 +- .../filesystem/SequenceFileRepositoryImplTest.java | 2 +- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index b46d51c1dba..f3131500852 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -28,7 +28,7 @@ public class SequenceFilePairConcatenatorTest { @Before public void setUp() { - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); concat = new SequenceFilePairConcatenator(iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 2a4b41336d3..2bdf1679d7a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -28,7 +28,7 @@ public class SingleEndSequenceFileConcatenatorTest { @Before public void setUp() { - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); concat = new SingleEndSequenceFileConcatenator(iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index 1670119d15c..fa09ddeff67 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -31,7 +31,7 @@ public class ChecksumFileProcessorTest { @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 7ee86c5f09c..e28e636b520 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -16,20 +16,14 @@ import java.nio.file.Path; import java.util.Iterator; -import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.util.ReflectionUtils; import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; @@ -65,7 +59,7 @@ public void setUp() { messageSource = mock(MessageSource.class); sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index b4768efbabf..240681616b5 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -41,7 +41,7 @@ public class GzipFileProcessorTest { @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index 4d7d78f8762..0bf8dec2123 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -39,7 +39,7 @@ public class SequenceFileRepositoryImplTest { public void setUp() throws IOException { baseDirectory = Files.createTempDirectory(TEMP_FILE_PREFIX); entityManager = mock(EntityManager.class); - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageService); } From 61de952cd33ca845f0677268ab66e9cd6c7c8995 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 13:49:57 -0500 Subject: [PATCH 045/655] Fixed doc testing (removed imports not needed and added missing comments). Moved sequencefilelistener and genomeassemblylistener to a common file as both were providing access to the iridafilestorageservice in an entity on @postload --- .../irida/model/assembly/GenomeAssembly.java | 4 +- .../model/sequenceFile/SequenceFile.java | 4 +- .../SequencingObjectConcatenator.java | 7 --- .../listeners/GenomeAssemblyListener.java | 29 ----------- .../listeners/IridaFileStorageListener.java | 52 +++++++++++++++++++ .../listeners/SequenceFileListener.java | 32 ------------ .../filesystem/IridaFileStorageService.java | 2 + .../ProjectReferenceFileController.java | 2 - 8 files changed, 58 insertions(+), 74 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 36e8d588533..81ffa146d4c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -20,7 +20,7 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; import ca.corefacility.bioinformatics.irida.model.joins.impl.SampleGenomeAssemblyJoin; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.GenomeAssemblyListener; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -32,7 +32,7 @@ @Entity @Table(name = "genome_assembly") @Inheritance(strategy = InheritanceType.JOINED) -@EntityListeners({ GenomeAssemblyListener.class }) +@EntityListeners({ IridaFileStorageListener.class }) @Audited public abstract class GenomeAssembly extends IridaResourceSupport implements IridaThing, IridaSequenceFile, VersionedFileFields { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 22c681b2d4d..70787389319 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -26,7 +26,7 @@ import ca.corefacility.bioinformatics.irida.model.remote.RemoteSynchronizable; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.SequenceFileListener; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; @@ -42,7 +42,7 @@ @Entity @Table(name = "sequence_file") @Audited -@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, SequenceFileListener.class }) +@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageListener.class }) public class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, VersionedFileFields, IridaSequenceFile, RemoteSynchronizable { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index d8b11a718e9..2294c83b3b2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -1,16 +1,9 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.List; -import java.util.Optional; /** * Class to concatenate multiple {@link SequencingObject}s and return a single new {@link SequencingObject}. This class diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java deleted file mode 100644 index f6741476f5a..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/GenomeAssemblyListener.java +++ /dev/null @@ -1,29 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; - -import javax.persistence.PostLoad; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.context.support.SpringBeanAutowiringSupport; - -import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; - -@Component -public class GenomeAssemblyListener { - - @Autowired - private IridaFileStorageService iridaFileStorageService; - - /** - * After the GenomeAssembly entity is loaded this method will provide - * the entity access to the iridaFileStorageService - * - * @param genomeAssembly The entity to provide the iridaFileStorageService to - */ - @PostLoad - public void afterAssemblyFileLoad(GenomeAssembly genomeAssembly) { - SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); - genomeAssembly.setIridaFileStorageService(iridaFileStorageService); - } -} \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java new file mode 100644 index 00000000000..38a0d535edf --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java @@ -0,0 +1,52 @@ +package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import javax.persistence.PostLoad; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; + +import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + +/** + * Component implementation to run on an entity after it is has been accessed from the db. + */ +@Component +public class IridaFileStorageListener { + private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageListener.class); + + @Autowired + private IridaFileStorageService iridaFileStorageService; + + /** + * After the entity is loaded this method will provide + * the entity access to the iridaFileStorageService + * + * @param fileSystemEntity The entity to provide the iridaFileStorageService to + */ + @PostLoad + public void afterEntityLoad(final VersionedFileFields fileSystemEntity) { + try { + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); + // Use reflection to get the setIridaFileStorageService method, make it accessible, and invoke it + Method iridaFileStorageServiceSetter = fileSystemEntity.getClass() + .getMethod("setIridaFileStorageService", IridaFileStorageService.class); + iridaFileStorageServiceSetter.setAccessible(true); + iridaFileStorageServiceSetter.invoke(fileSystemEntity, iridaFileStorageService); + } catch (NoSuchMethodException e) { + logger.error("The specified method does not exist. " + e.getMessage()); + } catch (IllegalAccessException e) { + logger.error(e.getMessage()); + } catch (InvocationTargetException e) { + logger.error(e.getMessage()); + } + } +} \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java deleted file mode 100644 index f95d3185f53..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/SequenceFileListener.java +++ /dev/null @@ -1,32 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; - -import javax.persistence.PostLoad; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.context.support.SpringBeanAutowiringSupport; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; - -/** - * Component implementation to run on an entity after it is has been accessed from the db. - */ -@Component -public class SequenceFileListener { - - @Autowired - private IridaFileStorageService iridaFileStorageService; - - /** - * After the SequenceFile entity is loaded this method will provide - * the entity access to the iridaFileStorageService - * - * @param sequenceFile The entity to provide the iridaFileStorageService to - */ - @PostLoad - public void afterSequenceFileLoad(SequenceFile sequenceFile) { - SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); - sequenceFile.setIridaFileStorageService(iridaFileStorageService); - } -} \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java index 6723d50ed21..219797eac38 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageService.java @@ -40,6 +40,8 @@ public interface IridaFileStorageService { * * @param source The {@link Path} to the file * @param target The {@link Path} to where file should be moved + * @param sequenceFileDir The {@link Path} to sequence file directory + * @param sequenceFileDirWithRevision The {@link Path} to sequence file revision directory */ public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java index e8faf32e111..6b9017c89f3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java @@ -1,7 +1,5 @@ package ca.corefacility.bioinformatics.irida.ria.web.projects.settings; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.security.Principal; import java.util.ArrayList; From 3a7a9e26522c85bd89358029f453440cf4024e94 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 14:37:40 -0500 Subject: [PATCH 046/655] Removed unused imports --- .../repositories/entity/listeners/IridaFileStorageListener.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java index 38a0d535edf..81c773c426e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java @@ -12,8 +12,6 @@ import org.springframework.web.context.support.SpringBeanAutowiringSupport; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** From f3429d99afc7a0bd179ae7ecbaeb31ffbc33610e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 22 Apr 2020 14:50:39 -0500 Subject: [PATCH 047/655] Removed unused imports --- .../repositories/entity/listeners/IridaFileStorageListener.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java index 38a0d535edf..81c773c426e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java @@ -12,8 +12,6 @@ import org.springframework.web.context.support.SpringBeanAutowiringSupport; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** From 9ce1be7b8f077b7854d2e2acde4a1f34b9e1ef74 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 23 Apr 2020 10:52:33 -0500 Subject: [PATCH 048/655] Upped the version of azure to fix race condition when opening blob stream. Updated comments and refactored code --- pom.xml | 3 ++- .../IridaFileStorageAzureServiceImpl.java | 25 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 14179a88253..d217bf50ef1 100644 --- a/pom.xml +++ b/pom.xml @@ -681,7 +681,7 @@ com.azure azure-storage-blob - 12.0.0 + ${azure.storage.blob.version} jakarta.xml.bind @@ -1208,6 +1208,7 @@ v1.21.1 1.0 2.8.6 + 12.6.0 UTF-8 diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index ce9a57fe47f..84cf8eb8096 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -67,7 +67,7 @@ public File getTemporaryFile(Path file) { blobClient.downloadToFile(filePath); fileToProcess = new File(filePath); } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); + logger.trace("Couldn't find file on azure [" + e + "]"); } return fileToProcess; @@ -84,7 +84,7 @@ public Long getFileSize(Path file) { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = blobClient.getProperties().getBlobSize(); } catch (BlobStorageException e) { - logger.trace("Couldn't calculate size as the file was not found [" + e + "]"); + logger.trace("Couldn't calculate size as the file was not found on azure [" + e + "]"); } return fileSize; } @@ -96,10 +96,13 @@ public Long getFileSize(Path file) { public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); - - logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); - blobClient.uploadFromFile(source.toString(), false); - logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); + try { + logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); + blobClient.uploadFromFile(source.toString(), false); + logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); + } catch (BlobStorageException e) { + logger.trace("Unable to upload file to azure [" + e + "]"); + } } /** @@ -144,7 +147,7 @@ public String getFileName(Path file) { .split("/"); fileName = blobNameTokens[blobNameTokens.length - 1]; } catch (BlobStorageException e) { - logger.trace("Couldn't find file [" + e + "]"); + logger.trace("Couldn't find file on azure [" + e + "]"); } return fileName; @@ -156,7 +159,7 @@ public String getFileName(Path file) { @Override public boolean fileExists(Path file) { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - if(blobClient.getProperties().getBlobSize() > 0) { + if(blobClient.exists()) { return true; } return false; @@ -167,7 +170,11 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + try { + blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + } catch (BlobStorageException e) { + logger.trace("Couldn't read file from azure [" + e + "]"); + } return blobClient.openInputStream(); } From 8f4d2f248caa9e2609acbacf1b21f9c5449a40ca Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 24 Apr 2020 15:28:10 -0500 Subject: [PATCH 049/655] Updated galaxylibraryservice to have access to files in cloud storage. --- .../analysis/ExecutionManagerConfig.java | 6 +++- .../upload/galaxy/GalaxyLibrariesService.java | 15 ++++++---- .../web/analysis/AnalysisAjaxController.java | 28 ++++++++----------- .../config/IridaApiNoGalaxyTestConfig.java | 10 +++++-- .../analysis/GalaxyExecutionTestConfig.java | 6 +++- .../integration/GalaxyHistoriesServiceIT.java | 9 ++++-- .../integration/GalaxyLibrariesServiceIT.java | 10 +++++-- .../galaxy/integration/GalaxyWorkflowsIT.java | 10 +++++-- .../unit/GalaxyLibrariesServiceTest.java | 21 ++++++++------ .../analysis/AnalysisAjaxControllerTest.java | 6 +++- .../files/ReferenceFileControllerTest.java | 3 +- .../web/files/SequenceFileControllerTest.java | 3 +- .../web/samples/SamplesControllerTest.java | 3 ++ .../AnalysisWorkspaceServiceGalaxyIT.java | 8 +++++- 14 files changed, 93 insertions(+), 45 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java index a7fbf069d6e..448c3672c62 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java @@ -29,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; import com.github.jmchilton.blend4j.galaxy.GalaxyInstanceFactory; @@ -89,6 +90,9 @@ public class ExecutionManagerConfig { @Autowired private Validator validator; + + @Autowired + private IridaFileStorageService iridaFileStorageService; /** * Builds a new ExecutionManagerGalaxy from the given properties. @@ -285,7 +289,7 @@ public GalaxyHistoriesService galaxyHistoriesService() throws ExecutionManagerCo @Lazy @Bean public GalaxyLibrariesService galaxyLibrariesService() throws ExecutionManagerConfigurationException { - return new GalaxyLibrariesService(librariesClient(), pollingTime, libraryTimeout, libraryUploadThreads); + return new GalaxyLibrariesService(librariesClient(), pollingTime, libraryTimeout, libraryUploadThreads, iridaFileStorageService); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index eef8dfb4cf4..8fbc4dc5aac 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -6,6 +6,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.List; @@ -30,6 +31,7 @@ import ca.corefacility.bioinformatics.irida.model.upload.galaxy.GalaxyProjectName; import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.github.jmchilton.blend4j.galaxy.LibrariesClient; @@ -58,6 +60,8 @@ public class GalaxyLibrariesService { private final int libraryPollingTime; private final int libraryUploadTimeout; + private IridaFileStorageService iridaFileStorageService; + /** * State a library dataset should be in on proper upload. */ @@ -83,7 +87,7 @@ public class GalaxyLibrariesService { * The thread pool size for parallel polling of Galaxy to check if uploads are finished. */ public GalaxyLibrariesService(LibrariesClient librariesClient, final int libraryPollingTime, - final int libraryUploadTimeout, final int threadPoolSize) { + final int libraryUploadTimeout, final int threadPoolSize, IridaFileStorageService iridaFileStorageService) { checkNotNull(librariesClient, "librariesClient is null"); checkArgument(libraryPollingTime > 0, "libraryPollingTime=" + libraryPollingTime + " must be positive"); checkArgument(libraryUploadTimeout > 0, "libraryUploadTimeout=" + libraryUploadTimeout + " must be positive"); @@ -97,6 +101,7 @@ public GalaxyLibrariesService(LibrariesClient librariesClient, final int library this.librariesClient = librariesClient; this.libraryPollingTime = libraryPollingTime; this.libraryUploadTimeout = libraryUploadTimeout; + this.iridaFileStorageService = iridaFileStorageService; executor = Executors.newFixedThreadPool(threadPoolSize); } @@ -151,9 +156,9 @@ public String fileToLibrary(Path path, InputFileType fileType, checkNotNull(fileType, "fileType is null"); checkNotNull(library, "library is null"); checkNotNull(library.getId(), "library id is null"); - checkState(path.toFile().exists(), "path " + path + " does not exist"); + checkState(iridaFileStorageService.fileExists(path), "path " + path + " does not exist"); - File file = path.toFile(); + File file = iridaFileStorageService.getTemporaryFile(path); try { LibraryContent rootContent = librariesClient.getRootFolder(library @@ -263,9 +268,9 @@ public Void call() throws Exception { * @throws IOException If there was an error reading the file to determine the file type. */ private InputFileType getFileType(Path path) throws IOException { - checkArgument(path.toFile().exists(), "path=[" + path + "] does not exist"); + checkArgument(iridaFileStorageService.fileExists(path), "path=[" + path + "] does not exist"); - return (FileUtils.isGzipped(path) ? InputFileType.FASTQ_SANGER_GZ : InputFileType.FASTQ_SANGER); + return (iridaFileStorageService.isGzipped(path) ? InputFileType.FASTQ_SANGER_GZ : InputFileType.FASTQ_SANGER); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 7733e58428d..702777edd92 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -42,6 +42,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.model.workflow.submission.ProjectAnalysisSubmissionJoin; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.analysis.auditing.AnalysisAudit; import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.*; @@ -93,6 +94,7 @@ public class AnalysisAjaxController { private UpdateAnalysisSubmissionPermission updateAnalysisPermission; private ExecutionManagerConfig configFile; private AnalysisAudit analysisAudit; + private IridaFileStorageService iridaFileStorageService; @Autowired public AnalysisAjaxController(AnalysisSubmissionService analysisSubmissionService, @@ -100,7 +102,7 @@ public AnalysisAjaxController(AnalysisSubmissionService analysisSubmissionServic ProjectService projectService, UpdateAnalysisSubmissionPermission updateAnalysisPermission, MetadataTemplateService metadataTemplateService, SequencingObjectService sequencingObjectService, AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor, - AnalysisOutputFileDownloadManager analysisOutputFileDownloadManager, MessageSource messageSource, ExecutionManagerConfig configFile, AnalysisAudit analysisAudit) { + AnalysisOutputFileDownloadManager analysisOutputFileDownloadManager, MessageSource messageSource, ExecutionManagerConfig configFile, AnalysisAudit analysisAudit, IridaFileStorageService iridaFileStorageService) { this.analysisSubmissionService = analysisSubmissionService; this.workflowsService = iridaWorkflowsService; @@ -115,6 +117,7 @@ public AnalysisAjaxController(AnalysisSubmissionService analysisSubmissionServic this.updateAnalysisPermission = updateAnalysisPermission; this.configFile = configFile; this.analysisAudit = analysisAudit; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -420,9 +423,8 @@ private AnalysisOutputFileInfo getAnalysisOutputFileInfo(AnalysisSubmission subm AnalysisOutputFileInfo info = new AnalysisOutputFileInfo(aof.getId(), submission.getId(), analysis.getId(), aof.getFile() .getFileName() - .toString(), fileExt, aof.getFile() - .toFile() - .length(), tool.getToolName(), tool.getToolVersion(), outputName); + .toString(), fileExt, iridaFileStorageService.getFileSize(aof.getFile()), + tool.getToolName(), tool.getToolVersion(), outputName); if (FILE_EXT_READ_FIRST_LINE.contains(fileExt)) { addFirstLine(info, aof); @@ -443,7 +445,7 @@ private void addFirstLine(AnalysisOutputFileInfo info, AnalysisOutputFile aof) { RandomAccessFile reader = null; final Path aofFile = aof.getFile(); try { - reader = new RandomAccessFile(aofFile.toFile(), "r"); + reader = new RandomAccessFile(iridaFileStorageService.getTemporaryFile(aofFile), "r"); info.setFirstLine(reader.readLine()); info.setFilePointer(reader.getFilePointer()); } catch (FileNotFoundException e) { @@ -499,13 +501,11 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable .toString()); contents.setFileExt(FileUtilities.getFileExt(aofFile.getFileName() .toString())); - contents.setFileSizeBytes(aof.getFile() - .toFile() - .length()); + contents.setFileSizeBytes(iridaFileStorageService.getFileSize(aofFile)); contents.setToolName(tool.getToolName()); contents.setToolVersion(tool.getToolVersion()); try { - final File file = aofFile.toFile(); + final File file = iridaFileStorageService.getTemporaryFile(aofFile); final RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r"); randomAccessFile.seek(seek); if (seek == 0) { @@ -741,7 +741,7 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { .getFile(); try { - String json = new Scanner(new BufferedReader(new FileReader(path.toFile()))).useDelimiter("\\Z") + String json = new Scanner(new BufferedReader(new FileReader(iridaFileStorageService.getTemporaryFile(path)))).useDelimiter("\\Z") .next(); // verify file is proper json file and map to a SistrResult list @@ -902,9 +902,7 @@ public ResponseEntity getImageFile(@PathVariable Long submissionId, Stri try { for (AnalysisOutputFile file : files) { - if (file.getFile() - .toFile() - .getName() + if (iridaFileStorageService.getFileName(file.getFile()) .contains(filename)) { outputFile = file; break; @@ -1089,9 +1087,7 @@ public ExcelData parseExcelFile(@PathVariable Long submissionId, String filename AnalysisOutputFile outputFile = null; for (AnalysisOutputFile file : files) { - if (file.getFile() - .toFile() - .getName() + if (iridaFileStorageService.getFileName(file.getFile()) .contains(filename)) { outputFile = file; break; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java index 056d75be9e5..328ce361f06 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java @@ -8,17 +8,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Profile; import org.springframework.scheduling.annotation.AsyncResult; +import ca.corefacility.bioinformatics.irida.config.services.IridaApiServicesConfig; import ca.corefacility.bioinformatics.irida.config.workflow.IridaWorkflowsTestConfig; import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.service.analysis.execution.galaxy.AnalysisExecutionServiceGalaxyCleanupAsync; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; @@ -57,11 +60,14 @@ * */ @Configuration -@Import({ IridaWorkflowsTestConfig.class }) +@Import({ IridaWorkflowsTestConfig.class, IridaApiServicesConfig.class }) @Profile("test") public class IridaApiNoGalaxyTestConfig { private static final Logger logger = LoggerFactory.getLogger(IridaApiNoGalaxyTestConfig.class); + @Autowired + private IridaFileStorageService iridaFileStorageService; + /** * @return An ExecutorService executing code in the same thread for testing * purposes. @@ -92,7 +98,7 @@ public GalaxyHistoriesService galaxyHistoriesService(HistoriesClient historiesCl @Bean public GalaxyLibrariesService galaxyLibrariesService(LibrariesClient librariesClient) { - return new GalaxyLibrariesService(librariesClient, 5, 60, 1); + return new GalaxyLibrariesService(librariesClient, 5, 60, 1, iridaFileStorageService); } @Bean diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java index ae262826e3e..aa38c4eefcf 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java @@ -15,6 +15,7 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.*; @@ -29,6 +30,9 @@ public class GalaxyExecutionTestConfig { @Autowired private LocalGalaxy localGalaxy; + @Autowired + private IridaFileStorageService iridaFileStorageService; + /** * Timeout in seconds to stop polling a Galaxy library. */ @@ -58,7 +62,7 @@ public GalaxyHistoriesService galaxyHistoriesService() { @Bean public GalaxyLibrariesService galaxyLibrariesService() { LibrariesClient librariesClient = localGalaxy.getGalaxyInstanceAdmin().getLibrariesClient(); - return new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); + return new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java index d9d135c03d7..32d0c7feaf8 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java @@ -43,6 +43,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; import com.github.jmchilton.blend4j.galaxy.GalaxyResponseException; @@ -83,7 +85,8 @@ public class GalaxyHistoriesServiceIT { private GalaxyInstance galaxyInstanceAdmin; private GalaxyLibrariesService galaxyLibrariesService; private HistoriesClient historiesClient; - + private IridaFileStorageService iridaFileStorageService; + private Path dataFile; private Path dataFile2; private Path dataFileCompressed; @@ -118,7 +121,9 @@ public void setup() throws URISyntaxException, IOException, CreateLibraryExcepti historiesClient = galaxyInstanceAdmin.getHistoriesClient(); ToolsClient toolsClient = galaxyInstanceAdmin.getToolsClient(); LibrariesClient librariesClient = galaxyInstanceAdmin.getLibrariesClient(); - galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); + + galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java index a7a4521de9a..55fbc95edc4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java @@ -32,6 +32,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; import com.github.jmchilton.blend4j.galaxy.LibrariesClient; @@ -64,6 +66,7 @@ public class GalaxyLibrariesServiceIT { private GalaxyInstance galaxyInstanceAdmin; private LibrariesClient librariesClient; + private IridaFileStorageService iridaFileStorageService; private static final InputFileType FILE_TYPE = InputFileType.FASTQ_SANGER; @@ -87,8 +90,9 @@ public class GalaxyLibrariesServiceIT { public void setup() throws URISyntaxException, IOException { galaxyInstanceAdmin = localGalaxy.getGalaxyInstanceAdmin(); librariesClient = galaxyInstanceAdmin.getLibrariesClient(); - - galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); + + galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); dataFile = Paths.get(GalaxyLibrariesServiceIT.class.getResource( "testData1.fastq").toURI()); @@ -208,7 +212,7 @@ public void testFilesToLibraryWaitFail() */ @Test(expected = UploadTimeoutException.class) public void testFilesToLibraryWaitFailTimeout() throws UploadException, GalaxyDatasetException { - galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, 1, 2, 1); + galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageService); Library library = buildEmptyLibrary("testFilesToLibraryWaitFailTimeout"); galaxyLibrariesService.filesToLibraryWait(Sets.newHashSet(dataFile, dataFile2), library, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java index 0609614fd57..ca4ccecc052 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java @@ -47,6 +47,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; @@ -101,7 +103,8 @@ public class GalaxyWorkflowsIT { private LibrariesClient librariesClient; private GalaxyWorkflowService galaxyWorkflowService; private GalaxyHistoriesService galaxyHistory; - + private IridaFileStorageService iridaFileStorageService; + private static final InputFileType FILE_TYPE = InputFileType.FASTQ_SANGER; private static final InputFileType INVALID_FILE_TYPE = null; @@ -160,8 +163,9 @@ public void setup() throws URISyntaxException, IOException { workflowsClient = galaxyAdminInstance.getWorkflowsClient(); historiesClient = galaxyAdminInstance.getHistoriesClient(); librariesClient = galaxyAdminInstance.getLibrariesClient(); - - GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); + + GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); galaxyWorkflowService = new GalaxyWorkflowService(workflowsClient, StandardCharsets.UTF_8); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java index 44f6f870005..7f987c58fc3 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java @@ -16,6 +16,8 @@ import ca.corefacility.bioinformatics.irida.exceptions.galaxy.CreateLibraryException; import ca.corefacility.bioinformatics.irida.model.upload.galaxy.GalaxyProjectName; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** * Tests for {@link GalaxyLibrariesService}. @@ -29,13 +31,16 @@ public class GalaxyLibrariesServiceTest { private final static String LIBRARY_ID = "1"; private Library testLibrary; - + + private IridaFileStorageService iridaFileStorageService; + /** * Setup for tests. */ @Before public void setup() { MockitoAnnotations.initMocks(this); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); setupLibrariesTest(); } @@ -54,7 +59,7 @@ private void setupLibrariesTest() { */ @Test(expected=IllegalArgumentException.class) public void testZeroPollingTime() { - new GalaxyLibrariesService(librariesClient, 0, 1, 1); + new GalaxyLibrariesService(librariesClient, 0, 1, 1, iridaFileStorageService); } /** @@ -62,7 +67,7 @@ public void testZeroPollingTime() { */ @Test(expected=IllegalArgumentException.class) public void testZeroUploadTimeout() { - new GalaxyLibrariesService(librariesClient, 1, 0, 1); + new GalaxyLibrariesService(librariesClient, 1, 0, 1, iridaFileStorageService); } /** @@ -70,7 +75,7 @@ public void testZeroUploadTimeout() { */ @Test(expected=IllegalArgumentException.class) public void testEqualPollingTimeUploadTimeout() { - new GalaxyLibrariesService(librariesClient, 1, 1, 1); + new GalaxyLibrariesService(librariesClient, 1, 1, 1, iridaFileStorageService); } /** @@ -78,7 +83,7 @@ public void testEqualPollingTimeUploadTimeout() { */ @Test public void testSuccessfullTimeoutValues() { - new GalaxyLibrariesService(librariesClient, 1, 2, 1); + new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageService); } /** @@ -86,7 +91,7 @@ public void testSuccessfullTimeoutValues() { */ @Test(expected=IllegalArgumentException.class) public void testFailThreadValue() { - new GalaxyLibrariesService(librariesClient, 1, 2, 0); + new GalaxyLibrariesService(librariesClient, 1, 2, 0, iridaFileStorageService); } /** @@ -98,7 +103,7 @@ public void testBuildEmptyLibrary() throws CreateLibraryException { when(librariesClient.createLibrary(any(Library.class))).thenReturn( testLibrary); - Library library = new GalaxyLibrariesService(librariesClient, 1, 2, 1).buildEmptyLibrary(new GalaxyProjectName( + Library library = new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageService).buildEmptyLibrary(new GalaxyProjectName( "test")); assertNotNull(library); @@ -115,6 +120,6 @@ public void testBuildEmptyLibrary() throws CreateLibraryException { public void testBuildEmptyLibraryFail() throws CreateLibraryException { when(librariesClient.createLibrary(any(Library.class))).thenReturn(null); - new GalaxyLibrariesService(librariesClient, 1, 2, 1).buildEmptyLibrary(new GalaxyProjectName("test")); + new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageService).buildEmptyLibrary(new GalaxyProjectName("test")); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java index 7cd5cba63ac..303828acd26 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java @@ -28,6 +28,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.model.workflow.submission.ProjectAnalysisSubmissionJoin; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.analysis.AnalysisAjaxController; import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.*; @@ -74,6 +76,7 @@ public class AnalysisAjaxControllerTest { private EmailController emailControllerMock; private AnalysisAudit analysisAuditMock; private HttpServletResponse httpServletResponseMock; + private IridaFileStorageService iridaFileStorageService; /** * Analysis Output File key names from {@link TestDataFactory#constructAnalysis()} */ @@ -95,11 +98,12 @@ public void init() { MessageSource messageSourceMock = mock(MessageSource.class); analysisAuditMock = mock(AnalysisAudit.class); httpServletResponseMock = mock(HttpServletResponse.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); analysisAjaxController = new AnalysisAjaxController(analysisSubmissionServiceMock, iridaWorkflowsServiceMock, userServiceMock, sampleService, projectServiceMock, updatePermission, metadataTemplateService, sequencingObjectService, analysisSubmissionSampleProcessor, - analysisOutputFileDownloadManager, messageSourceMock, configFileMock, analysisAuditMock); + analysisOutputFileDownloadManager, messageSourceMock, configFileMock, analysisAuditMock, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java index 41da238759f..fe315e9473d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java @@ -29,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ProjectReferenceFileJoin; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.files.ReferenceFileController; @@ -63,7 +64,7 @@ public void setUp() { projectService = mock(ProjectService.class); referenceFileService = mock(ReferenceFileService.class); messageSource = mock(MessageSource.class); - iridaFileStorageService = mock(IridaFileStorageService.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); // Set up the reference file Path path = Paths.get(FILE_PATH); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index 15947f8aa3f..10ebde638ea 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -22,6 +22,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; import ca.corefacility.bioinformatics.irida.service.AnalysisService; @@ -50,7 +51,7 @@ public void setUp() { sequencingRunService = mock(SequencingRunService.class); analysisService = mock(AnalysisService.class); objectService = mock(SequencingObjectService.class); - iridaFileStorageService = mock(IridaFileStorageService.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); controller = new SequenceFileController(objectService, sequencingRunService, analysisService, iridaFileStorageService); Path path = Paths.get(FILE_PATH); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index 1bbc14aba1a..8164b0e1f40 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -30,6 +30,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; @@ -75,6 +76,8 @@ public void setUp() { messageSource = mock(MessageSource.class); updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); + controller = new SamplesController(sampleService, projectService, sequencingObjectService, updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource, iridaFileStorageService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java index 5ccd78dac03..7b8de89ef9f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java @@ -82,6 +82,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.Util; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; @@ -184,6 +186,8 @@ public class AnalysisWorkspaceServiceGalaxyIT { private static final String SAMPLE1_NAME = "sample1"; + private IridaFileStorageService iridaFileStorageService; + /** * Sets up variables for testing. * @@ -227,7 +231,9 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc HistoriesClient historiesClient = galaxyInstanceAdmin.getHistoriesClient(); ToolsClient toolsClient = galaxyInstanceAdmin.getToolsClient(); LibrariesClient librariesClient = galaxyInstanceAdmin.getLibrariesClient(); - GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); + + GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); galaxyHistoriesService = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); From b3778b5084546bbc9be73a3c1bda6e97879fdbd0 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 27 Apr 2020 14:33:52 -0500 Subject: [PATCH 050/655] Updated tests --- .../irida/ria/integration/samples/SampleDetailsPageIT.java | 2 +- .../ria/integration/sequenceFiles/SequenceFilePageIT.java | 2 +- .../unit/web/analysis/AnalysesTableAjaxControllerTest.java | 6 ++++-- .../controller/test/integration/security/SecurityIT.java | 4 +++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java index 9b1c5af3c78..b46896841d9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java @@ -18,7 +18,7 @@ @DatabaseSetup("/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplePagesIT.xml") public class SampleDetailsPageIT extends AbstractIridaUIITChromeDriver { private static final Long SAMPLE_ID = 1L; - private static final String SAMPLE_CREATED_DATE = "18 Jul 2013"; + private static final String SAMPLE_CREATED_DATE = "18 Jul. 2013"; private static final String SAMPLE_ORGANISM = "E. coli"; private static final String SAMPLE_LATITUDE = "49.8994"; private static final String SAMPLE_LONGITUDE = "-97.1392"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index 78cd6f15327..e8410e6de88 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul 18, 2013"; + private static final String FILE_CREATED = "Jul. 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java index 26e9a0bb944..ba4d19c8e62 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java @@ -2,6 +2,8 @@ import java.security.Principal; +import javax.ws.rs.core.MediaType; + import org.junit.Before; import org.junit.Test; import org.mockito.InjectMocks; @@ -65,7 +67,7 @@ public void setUp() { @Test public void testGetAnalysisStates() throws Exception { - mockMvc.perform(get("/ajax/analyses/states").principal(mock(Principal.class))) + mockMvc.perform(get("/ajax/analyses/states").accept(MediaType.APPLICATION_JSON).principal(mock(Principal.class))) .andExpect(status().isOk()) .andExpect(jsonPath("$[0].value", is("NEW"))) .andExpect(jsonPath("$[1].value", is("PREPARING"))) @@ -81,7 +83,7 @@ public void testGetAnalysisStates() throws Exception { @Test public void testGetWorkflowTypes() throws Exception { - mockMvc.perform(get("/ajax/analyses/types").principal(mock(Principal.class))) + mockMvc.perform(get("/ajax/analyses/types").accept(MediaType.APPLICATION_JSON).principal(mock(Principal.class))) .andExpect(status().isOk()) .andExpect(jsonPath("$[0].value", is("Pipeline 1"))) .andExpect(jsonPath("$[1].value", is("Pipeline 2"))); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java index ee6fc5407af..da269e556d5 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java @@ -7,6 +7,8 @@ import org.apache.http.HttpStatus; import org.junit.Test; +import com.jayway.restassured.http.ContentType; + /** * General tests relating to security for the REST API. * @@ -19,7 +21,7 @@ public class SecurityIT { */ @Test public void testAccessWithoutAuthentication() { - expect().body("error", is("invalid_request")).and() + expect().contentType(ContentType.JSON).body("error", is("invalid_request")).and() .body("error_description", containsString("No client credentials were provided")) .statusCode(HttpStatus.SC_UNAUTHORIZED).when().get("/api"); } From ce85c264c2101c58af82305b22c3e0834d6f2aa3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 27 Apr 2020 16:50:26 -0500 Subject: [PATCH 051/655] Updated tests --- .../irida/ria/integration/samples/SampleDetailsPageIT.java | 2 +- .../ria/integration/sequenceFiles/SequenceFilePageIT.java | 2 +- .../unit/web/analysis/AnalysesTableAjaxControllerTest.java | 6 ++++-- .../controller/test/integration/security/SecurityIT.java | 4 +++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java index 9b1c5af3c78..b46896841d9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java @@ -18,7 +18,7 @@ @DatabaseSetup("/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplePagesIT.xml") public class SampleDetailsPageIT extends AbstractIridaUIITChromeDriver { private static final Long SAMPLE_ID = 1L; - private static final String SAMPLE_CREATED_DATE = "18 Jul 2013"; + private static final String SAMPLE_CREATED_DATE = "18 Jul. 2013"; private static final String SAMPLE_ORGANISM = "E. coli"; private static final String SAMPLE_LATITUDE = "49.8994"; private static final String SAMPLE_LONGITUDE = "-97.1392"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index 78cd6f15327..e8410e6de88 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul 18, 2013"; + private static final String FILE_CREATED = "Jul. 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java index 26e9a0bb944..ba4d19c8e62 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysesTableAjaxControllerTest.java @@ -2,6 +2,8 @@ import java.security.Principal; +import javax.ws.rs.core.MediaType; + import org.junit.Before; import org.junit.Test; import org.mockito.InjectMocks; @@ -65,7 +67,7 @@ public void setUp() { @Test public void testGetAnalysisStates() throws Exception { - mockMvc.perform(get("/ajax/analyses/states").principal(mock(Principal.class))) + mockMvc.perform(get("/ajax/analyses/states").accept(MediaType.APPLICATION_JSON).principal(mock(Principal.class))) .andExpect(status().isOk()) .andExpect(jsonPath("$[0].value", is("NEW"))) .andExpect(jsonPath("$[1].value", is("PREPARING"))) @@ -81,7 +83,7 @@ public void testGetAnalysisStates() throws Exception { @Test public void testGetWorkflowTypes() throws Exception { - mockMvc.perform(get("/ajax/analyses/types").principal(mock(Principal.class))) + mockMvc.perform(get("/ajax/analyses/types").accept(MediaType.APPLICATION_JSON).principal(mock(Principal.class))) .andExpect(status().isOk()) .andExpect(jsonPath("$[0].value", is("Pipeline 1"))) .andExpect(jsonPath("$[1].value", is("Pipeline 2"))); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java index ee6fc5407af..da269e556d5 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/integration/security/SecurityIT.java @@ -7,6 +7,8 @@ import org.apache.http.HttpStatus; import org.junit.Test; +import com.jayway.restassured.http.ContentType; + /** * General tests relating to security for the REST API. * @@ -19,7 +21,7 @@ public class SecurityIT { */ @Test public void testAccessWithoutAuthentication() { - expect().body("error", is("invalid_request")).and() + expect().contentType(ContentType.JSON).body("error", is("invalid_request")).and() .body("error_description", containsString("No client credentials were provided")) .statusCode(HttpStatus.SC_UNAUTHORIZED).when().get("/api"); } From ed8865f30d2231b6e77c329ff02e65cc8a936d72 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 28 Apr 2020 19:27:05 -0500 Subject: [PATCH 052/655] Updated code to work with newer version of jackson-core --- .../remote/impl/RemoteRepositoryImpl.java | 1 - .../web/controller/api/RESTRootController.java | 1 + .../api/projects/RESTProjectAnalysisController.java | 3 +++ .../api/projects/RESTProjectSamplesController.java | 13 +++++++++---- .../api/projects/RESTProjectUsersController.java | 8 ++++---- .../api/projects/RESTProjectsController.java | 2 ++ 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java index 62edd9adb6e..a4bee9ad3e3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java @@ -96,7 +96,6 @@ public List list(String uri, RemoteAPI remoteAPI) { public boolean getServiceStatus(RemoteAPI remoteAPI) { OAuthTokenRestTemplate restTemplate = new OAuthTokenRestTemplate(tokenService, remoteAPI); ResponseEntity forEntity = restTemplate.getForEntity(remoteAPI.getServiceURI(), String.class); - return forEntity.getStatusCode() == HttpStatus.OK; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java index 97abb77bd29..b755348303a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java @@ -85,6 +85,7 @@ public void initLinks() { * @return a response to the client. */ @RequestMapping(method = RequestMethod.GET, value = "/api") + @ResponseBody public ModelMap getLinks(final HttpServletRequest request) { logger.debug("Discovering application"); RootResource resource = new RootResource(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java index 48b731c8531..1a1bc52e2ba 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException; import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; @@ -65,6 +66,7 @@ public RESTProjectAnalysisController(ProjectService projectService, * {@link Project}. */ @RequestMapping(value = "/api/projects/{projectId}/analyses", method = RequestMethod.GET) + @ResponseBody public ModelMap getProjectAnalyses(@PathVariable Long projectId) { logger.debug("Loading analyses for project [" + projectId + "]"); @@ -107,6 +109,7 @@ public ModelMap getProjectAnalyses(@PathVariable Long projectId) { * found in IRIDA. */ @RequestMapping(value = "/api/projects/{projectId}/analyses/{type}", method = RequestMethod.GET) + @ResponseBody public ModelMap getProjectAnalysesByType(@PathVariable Long projectId, @PathVariable String type) throws IridaWorkflowNotFoundException { logger.debug("Loading analyses for project [" + projectId + "] by type [" + type + "]"); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java index 63a34419a4f..89a6827c303 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java @@ -21,10 +21,7 @@ import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.*; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.view.RedirectView; @@ -86,6 +83,7 @@ public RESTProjectSamplesController(ProjectService projectService, SampleService * project. */ @RequestMapping(value = "/api/projects/{projectId}/samples", method = RequestMethod.POST, consumes = "application/idcollection+json") + @ResponseBody public ModelMap copySampleToProject(final @PathVariable Long projectId, final @RequestBody List sampleIds, HttpServletResponse response) { @@ -140,6 +138,7 @@ public ModelMap copySampleToProject(final @PathVariable Long projectId, final @R * location information. */ @RequestMapping(value = "/api/projects/{projectId}/samples", method = RequestMethod.POST, consumes = "!application/idcollection+json") + @ResponseBody public ModelMap addSampleToProject(@PathVariable Long projectId, @RequestBody Sample sample, HttpServletResponse response) { ModelMap model = new ModelMap(); @@ -177,6 +176,7 @@ public ModelMap addSampleToProject(@PathVariable Long projectId, @RequestBody Sa * @return the list of {@link Sample}s associated with this {@link Project}. */ @RequestMapping(value = "/api/projects/{projectId}/samples", method = RequestMethod.GET) + @ResponseBody public ModelMap getProjectSamples(@PathVariable Long projectId) { ModelMap modelMap = new ModelMap(); @@ -206,6 +206,7 @@ public ModelMap getProjectSamples(@PathVariable Long projectId) { * @return The found sample */ @RequestMapping(value = "/api/projects/{projectId}/samples/bySequencerId/{seqeuncerId}", method = RequestMethod.GET) + @ResponseBody public ModelAndView getProjectSampleBySequencerId(@PathVariable Long projectId, @PathVariable String seqeuncerId) { Project p = projectService.read(projectId); @@ -233,6 +234,7 @@ public ModelAndView getProjectSampleBySequencerId(@PathVariable Long projectId, * @return a representation of the specific sample. */ @RequestMapping(value = "/api/projects/{projectId}/samples/{sampleId}", method = RequestMethod.GET) + @ResponseBody public ModelMap getProjectSample(@PathVariable Long projectId, @PathVariable Long sampleId) { // read project/sample to verify sample exists in project Project project = projectService.read(projectId); @@ -259,6 +261,7 @@ public ModelMap getProjectSample(@PathVariable Long projectId, @PathVariable Lon * @return representation of the sample */ @RequestMapping(value = "/api/samples/{sampleId}", method = RequestMethod.GET) + @ResponseBody public ModelMap getSample(@PathVariable Long sampleId) { ModelMap modelMap = new ModelMap(); Sample s = sampleService.read(sampleId); @@ -314,6 +317,7 @@ private void addLinksForSample(final Optional p, final Sample s) { * and collection of {@link Sample}. */ @RequestMapping(value = "/api/projects/{projectId}/samples/{sampleId}", method = RequestMethod.DELETE) + @ResponseBody public ModelMap removeSampleFromProject(@PathVariable Long projectId, @PathVariable Long sampleId) { ModelMap modelMap = new ModelMap(); @@ -350,6 +354,7 @@ public ModelMap removeSampleFromProject(@PathVariable Long projectId, @PathVaria */ @RequestMapping(value = "/api/samples/{sampleId}", method = RequestMethod.PATCH, consumes = { MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE }) + @ResponseBody public ModelMap updateSample(@PathVariable Long sampleId, @RequestBody Map updatedFields) { ModelMap modelMap = new ModelMap(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java index 644b6c63a92..c70aa9efaad 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java @@ -12,10 +12,7 @@ import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.*; import ca.corefacility.bioinformatics.irida.exceptions.ProjectWithoutOwnerException; import ca.corefacility.bioinformatics.irida.model.enums.ProjectRole; @@ -79,6 +76,7 @@ public RESTProjectUsersController(UserService userService, ProjectService projec * thrown in this method, but needs to be listed. */ @RequestMapping(value = "/api/projects/{projectId}/users", method = RequestMethod.GET) + @ResponseBody public ModelMap getUsersForProject(@PathVariable Long projectId) throws ProjectWithoutOwnerException { ResourceCollection resources = new ResourceCollection<>(); @@ -121,6 +119,7 @@ public ModelMap getUsersForProject(@PathVariable Long projectId) throws ProjectW * {@code linkTo} and {@code methodOn}. */ @RequestMapping(value = "/api/projects/{projectId}/users", method = RequestMethod.POST) + @ResponseBody public ModelMap addUserToProject(@PathVariable Long projectId, @RequestBody Map representation, HttpServletResponse response) throws ProjectWithoutOwnerException { // first, get the project @@ -177,6 +176,7 @@ public ModelMap addUserToProject(@PathVariable Long projectId, @RequestBody Map< * @throws ProjectWithoutOwnerException if removing this user will leave the project without an owner */ @RequestMapping(value = "/api/projects/{projectId}/users/{userId}", method = RequestMethod.DELETE) + @ResponseBody public ModelMap removeUserFromProject(@PathVariable Long projectId, @PathVariable String userId) throws ProjectWithoutOwnerException { // Read the project and user from the database diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java index 669de3f6329..da0f8e1046c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java @@ -12,6 +12,7 @@ import org.springframework.hateoas.Link; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; import ca.corefacility.bioinformatics.irida.exceptions.ProjectWithoutOwnerException; import ca.corefacility.bioinformatics.irida.model.project.Project; @@ -23,6 +24,7 @@ * */ @Controller +@ResponseBody @RequestMapping(value = "/api/projects") public class RESTProjectsController extends RESTGenericController { From b0a4b16df29a0c50f2c8cbd08c6d581d493ce7d0 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 29 Apr 2020 11:10:59 -0500 Subject: [PATCH 053/655] Removed unused imports --- .../irida/pipeline/upload/galaxy/GalaxyLibrariesService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index 8fbc4dc5aac..63982153b90 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -6,7 +6,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.List; @@ -32,7 +31,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; -import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.github.jmchilton.blend4j.galaxy.LibrariesClient; import com.github.jmchilton.blend4j.galaxy.beans.FilesystemPathsLibraryUpload; From 04dbb53007c8f1d26627bd4617795d8fabbe42dd Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 29 Apr 2020 11:31:43 -0500 Subject: [PATCH 054/655] Removed unused imports and added missing javadoc comment --- .../repositories/entity/listeners/AnalysisFastQCListener.java | 3 +++ .../filesystem/IridaFileStorageAzureServiceImpl.java | 1 - .../irida/ria/web/files/SequenceFileController.java | 1 - .../irida/ria/web/samples/SamplesController.java | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java index 08a4c06d672..3164fc12ad4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisFastQCListener.java @@ -9,6 +9,9 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +/** + * Component implementation to run on a AnalysisFastQC entity after it is has been accessed from the db. + */ @Component public class AnalysisFastQCListener { @Autowired diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 82c0424b8e9..711ea773f7f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Date; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java index d0be0c8063d..2b53659b72c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java @@ -3,7 +3,6 @@ import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Date; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 03c5cb590c5..74b5d110e79 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.ria.web.samples; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; From 6bf87a9fc5818777fe3442bf93b6d78f3f12ee2f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 29 Apr 2020 17:45:22 -0500 Subject: [PATCH 055/655] Updated galaxy connector code to make use of cloud stored reference files --- .../config/analysis/ExecutionManagerConfig.java | 2 +- .../upload/galaxy/GalaxyHistoriesService.java | 12 ++++++++---- .../irida/config/IridaApiNoGalaxyTestConfig.java | 2 +- .../config/analysis/GalaxyExecutionTestConfig.java | 2 +- .../galaxy/integration/GalaxyHistoriesServiceIT.java | 2 +- .../upload/galaxy/integration/GalaxyWorkflowsIT.java | 2 +- .../galaxy/unit/GalaxyHistoriesServiceTest.java | 8 ++++++-- .../AnalysisWorkspaceServiceGalaxyIT.java | 2 +- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java index 448c3672c62..d45a50e7326 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java @@ -279,7 +279,7 @@ public LibrariesClient librariesClient() throws ExecutionManagerConfigurationExc @Lazy @Bean public GalaxyHistoriesService galaxyHistoriesService() throws ExecutionManagerConfigurationException { - return new GalaxyHistoriesService(historiesClient(), toolsClient(), galaxyLibrariesService()); + return new GalaxyHistoriesService(historiesClient(), toolsClient(), galaxyLibrariesService(), iridaFileStorageService); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java index 02ad61d459f..a566444bd31 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java @@ -29,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.github.jmchilton.blend4j.galaxy.ToolsClient; @@ -63,7 +64,9 @@ public class GalaxyHistoriesService { private ToolsClient toolsClient; private GalaxyLibrariesService librariesService; - + + private IridaFileStorageService iridaFileStorageService; + /** * Builds a new GalaxyHistory object for working with Galaxy Histories. * @param historiesClient The HistoriesClient for interacting with Galaxy histories. @@ -71,7 +74,7 @@ public class GalaxyHistoriesService { * @param librariesService A service for dealing with Galaxy libraries. */ public GalaxyHistoriesService(HistoriesClient historiesClient, - ToolsClient toolsClient, GalaxyLibrariesService librariesService) { + ToolsClient toolsClient, GalaxyLibrariesService librariesService, IridaFileStorageService iridaFileStorageService) { checkNotNull(historiesClient, "historiesClient is null"); checkNotNull(toolsClient, "toolsClient is null"); checkNotNull(librariesService, "librariesService is null"); @@ -79,6 +82,7 @@ public GalaxyHistoriesService(HistoriesClient historiesClient, this.historiesClient = historiesClient; this.toolsClient = toolsClient; this.librariesService = librariesService; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -148,9 +152,9 @@ public Dataset fileToHistory(Path path, InputFileType fileType, History history) checkNotNull(fileType, "fileType is null"); checkNotNull(history, "history is null"); checkNotNull(history.getId(), "history id is null"); - checkState(path.toFile().exists(), "path " + path + " does not exist"); + checkState(iridaFileStorageService.fileExists(path), "path " + path + " does not exist"); - File file = path.toFile(); + File file = iridaFileStorageService.getTemporaryFile(path); FileUploadRequest uploadRequest = new FileUploadRequest(history.getId(), file); uploadRequest.setFileType(fileType.toString()); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java index 328ce361f06..c77d99fd79c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java @@ -93,7 +93,7 @@ public Future cleanupSubmission(AnalysisSubmission analysisS @Bean public GalaxyHistoriesService galaxyHistoriesService(HistoriesClient historiesClient, ToolsClient toolsClient, GalaxyLibrariesService librariesService) { - return new GalaxyHistoriesService(historiesClient, toolsClient, librariesService); + return new GalaxyHistoriesService(historiesClient, toolsClient, librariesService, iridaFileStorageService); } @Bean diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java index aa38c4eefcf..e039db6d9d3 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java @@ -55,7 +55,7 @@ public GalaxyInstance galaxyInstance() { public GalaxyHistoriesService galaxyHistoriesService() { HistoriesClient historiesClient = localGalaxy.getGalaxyInstanceAdmin().getHistoriesClient(); ToolsClient toolsClient = localGalaxy.getGalaxyInstanceAdmin().getToolsClient(); - return new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService()); + return new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService(), iridaFileStorageService); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java index 32d0c7feaf8..363eae53ba3 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java @@ -126,7 +126,7 @@ public void setup() throws URISyntaxException, IOException, CreateLibraryExcepti galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, - galaxyLibrariesService); + galaxyLibrariesService, iridaFileStorageService); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java index ca4ccecc052..a030292d119 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java @@ -166,7 +166,7 @@ public void setup() throws URISyntaxException, IOException { iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); - galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); + galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageService); galaxyWorkflowService = new GalaxyWorkflowService(workflowsClient, StandardCharsets.UTF_8); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java index ded13850a35..3560bba0b18 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java @@ -33,6 +33,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.Util; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.github.jmchilton.blend4j.galaxy.ToolsClient; @@ -61,6 +63,7 @@ public class GalaxyHistoriesServiceTest { @Mock private ClientResponse invalidResponse; @Mock private ClientResponse okayResponse; @Mock private GalaxyLibrariesService galaxyLibrariesService; + private IridaFileStorageService iridaFileStorageService; private GalaxyHistoriesService galaxyHistory; @@ -97,9 +100,10 @@ public void setup() throws URISyntaxException { ClientResponse.Status.OK); when(invalidResponse.getClientResponseStatus()).thenReturn( ClientResponse.Status.FORBIDDEN); - + iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); + galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, - galaxyLibrariesService); + galaxyLibrariesService, iridaFileStorageService); dataFile = Paths.get(this.getClass().getResource("testData1.fastq").toURI()); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java index 7b8de89ef9f..6a34ccbad18 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java @@ -235,7 +235,7 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageService); - galaxyHistoriesService = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); + galaxyHistoriesService = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageService); pairSequenceFiles1A = new ArrayList<>(); pairSequenceFiles1A.add(sequenceFilePathA); From 544187bd7922427987db5dedce10f5b2059d3971 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 30 Apr 2020 12:14:48 -0500 Subject: [PATCH 056/655] Updated downloading of analysis outputs as a zipped folder with cloud stored files --- .../workflow/analysis/AnalysisOutputFile.java | 12 +++- .../listeners/AnalysisOutputListener.java | 32 ++++++++++ .../IridaFileStorageAzureServiceImpl.java | 16 +++-- .../irida/ria/utilities/FileUtilities.java | 64 +++++++++---------- .../analysis/AnalysesTableAjaxController.java | 7 +- .../web/analysis/AnalysisAjaxController.java | 13 ++-- 6 files changed, 93 insertions(+), 51 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisOutputListener.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 8888163ffdd..c9bb99f65a5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -26,8 +26,10 @@ import ca.corefacility.bioinformatics.irida.model.IridaResourceSupport; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.AnalysisOutputListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; /** * Store file references to files produced by a workflow execution that we @@ -37,7 +39,7 @@ */ @Entity @Table(name = "analysis_output_file") -@EntityListeners(RelativePathTranslatorListener.class) +@EntityListeners({RelativePathTranslatorListener.class, AnalysisOutputListener.class }) public class AnalysisOutputFile extends IridaResourceSupport implements IridaThing, VersionedFileFields { @Id @@ -66,6 +68,8 @@ public class AnalysisOutputFile extends IridaResourceSupport implements IridaThi @Column(name = "label_prefix") private final String labelPrefix; + private static IridaFileStorageService iridaFileStorageService; + /** * for hibernate */ @@ -178,6 +182,10 @@ public boolean equals(Object o) { return false; } + public void setIridaFileStorageService(IridaFileStorageService iridaFileStorageService) { + this.iridaFileStorageService = iridaFileStorageService; + } + /** * Read the bytes for an image output file * @@ -185,7 +193,7 @@ public boolean equals(Object o) { * @throws IOException if the file couldn't be read */ public byte[] getBytesForFile() throws IOException { - byte[] bytes = Files.readAllBytes(getFile()); + byte[] bytes = iridaFileStorageService.readAllBytes(getFile()); return bytes; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisOutputListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisOutputListener.java new file mode 100644 index 00000000000..bec17386418 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/AnalysisOutputListener.java @@ -0,0 +1,32 @@ +package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; + +import javax.persistence.PostLoad; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; + +import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; + +/** + * Component implementation to run on a AnalysisOutputFile entity after it is has been accessed from the db. + */ +@Component +public class AnalysisOutputListener { + @Autowired + private IridaFileStorageService iridaFileStorageService; + + /** + * After the AnalysisOutputFile entity is loaded this method will provide + * the entity access to the iridaFileStorageService + * + * @param analysisOutputFile The entity to provide the iridaFileStorageService to + */ + @PostLoad + public void afterAnalysisOutputLoad(AnalysisOutputFile analysisOutputFile) { + SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); + analysisOutputFile.setIridaFileStorageService(iridaFileStorageService); + } +} + diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 711ea773f7f..308ed7c72fa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Date; @@ -57,18 +58,21 @@ public File getTemporaryFile(Path file) { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { - // Create a file that will be unique in the /tmp/ folder. We append the current date/time - // to the file name - String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); // Since the file system is virtual the full file path is the file name. // We split it on "/" and get the last token which is the actual file name. + String [] blobNameTokens = blobClient.getBlobName().split("/"); String fileName = blobNameTokens[blobNameTokens.length-1]; - String filePath = tmpDir + fileName; - blobClient.downloadToFile(filePath); - fileToProcess = new File(filePath); + + Path targetDirectory = Files.createTempDirectory(null); + Path target = targetDirectory.resolve(fileName); + + blobClient.downloadToFile(target.toAbsolutePath().toString()); + fileToProcess = new File(target.toAbsolutePath().toString()); } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); + } catch (IOException e) { + logger.trace("Couldn't create temp directory [" + e + "]"); } return fileToProcess; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java index 8a561611167..8f5754cce44 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java @@ -2,7 +2,6 @@ import java.io.*; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -13,6 +12,7 @@ import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.apache.poi.ss.usermodel.*; import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; @@ -21,6 +21,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.ProjectSampleAnalysisOutputInfo; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.web.dto.ExcelCol; import ca.corefacility.bioinformatics.irida.ria.web.dto.ExcelData; import ca.corefacility.bioinformatics.irida.ria.web.dto.ExcelHeader; @@ -47,15 +48,13 @@ public class FileUtilities { * Utility method for download a zip file containing all output files from * an analysis. * - * @param response - * {@link HttpServletResponse} - * @param fileName - * Name fo the file to create - * @param files - * Set of {@link AnalysisOutputFile} + * @param response {@link HttpServletResponse} + * @param fileName Name fo the file to create + * @param files Set of {@link AnalysisOutputFile} + * @param iridaFileStorageService file storage implementation */ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse response, String fileName, - Set files) { + Set files, IridaFileStorageService iridaFileStorageService) { /* * Replacing spaces and commas as they cause issues with * Content-disposition response header. @@ -74,7 +73,7 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re ZipOutputStream outputStream = new ZipOutputStream(responseStream)) { for (AnalysisOutputFile file : files) { - if (!Files.exists(file.getFile())) { + if (!iridaFileStorageService.fileExists(file.getFile())) { response.setStatus(404); throw new FileNotFoundException(); } @@ -87,8 +86,7 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re outputStream.putNextEntry(new ZipEntry(zipEntryName.toString())); // 3) COPY all of thy bytes from the file to the output stream. - Files.copy(file.getFile(), outputStream); - + IOUtils.copy(iridaFileStorageService.getFileInputStream(file.getFile()),outputStream); // 4) Close the current entry in the archive in preparation for // the next entry. outputStream.closeEntry(); @@ -116,15 +114,14 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re /** * Utility method for download a zip file containing all output files from * an analysis. - * @param response - * {@link HttpServletResponse} - * @param fileName - * Name fo the file to create - * @param files - * Set of {@link AnalysisOutputFile} + * + * @param response {@link HttpServletResponse} + * @param fileName Name fo the file to create + * @param files Set of {@link AnalysisOutputFile} + * @param iridaFileStorageService file storage implementation */ public static void createBatchAnalysisOutputFileZippedResponse(HttpServletResponse response, String fileName, - Map files) { + Map files, IridaFileStorageService iridaFileStorageService) { /* * Replacing spaces and commas as they cause issues with * Content-disposition response header. @@ -142,9 +139,9 @@ public static void createBatchAnalysisOutputFileZippedResponse(HttpServletRespon for (Map.Entry entry : files.entrySet()) { final AnalysisOutputFile file = entry.getValue(); final ProjectSampleAnalysisOutputInfo outputInfo = entry.getKey(); - if (!Files.exists(file.getFile())) { + if (!iridaFileStorageService.fileExists(file.getFile())) { response.setStatus(404); - throw new FileNotFoundException("File '" + file.getFile().toFile().getAbsolutePath() + "' does not exist!"); + throw new FileNotFoundException("File '" + file.getFile().toAbsolutePath() + "' does not exist!"); } // 1) Build a folder/file name // building similar filename for each analysis output file as: @@ -159,7 +156,7 @@ public static void createBatchAnalysisOutputFileZippedResponse(HttpServletRespon outputStream.putNextEntry(new ZipEntry(fileName + "/" + outputFilename)); // 3) COPY all of thy bytes from the file to the output stream. - Files.copy(file.getFile(), outputStream); + IOUtils.copy(iridaFileStorageService.getFileInputStream(file.getFile()),outputStream); // 4) Close the current entry in the archive in preparation for // the next entry. @@ -182,13 +179,12 @@ public static void createBatchAnalysisOutputFileZippedResponse(HttpServletRespon /** * Utility method for download single file from an analysis. * - * @param response - * {@link HttpServletResponse} - * @param file - * Set of {@link AnalysisOutputFile} - * @param fileName Filename + * @param response {@link HttpServletResponse} + * @param file Set of {@link AnalysisOutputFile} + * @param fileName Filename + * @param iridaFileStorageService file storage implementation */ - public static void createSingleFileResponse(HttpServletResponse response, AnalysisOutputFile file, String fileName) { + public static void createSingleFileResponse(HttpServletResponse response, AnalysisOutputFile file, String fileName, IridaFileStorageService iridaFileStorageService) { fileName = formatName(fileName); // set the response headers before we do *ANYTHING* so that the filename @@ -197,7 +193,7 @@ public static void createSingleFileResponse(HttpServletResponse response, Analys response.setContentType(CONTENT_TYPE_TEXT); try (ServletOutputStream outputStream = response.getOutputStream()) { - Files.copy(file.getFile(), response.getOutputStream()); + IOUtils.copy(iridaFileStorageService.getFileInputStream(file.getFile()), response.getOutputStream()); } catch (IOException e) { // this generally means that the user has cancelled the download // from their web browser; we can safely ignore this @@ -208,18 +204,16 @@ public static void createSingleFileResponse(HttpServletResponse response, Analys } } - /** * Utility method for download single file from an analysis. * - * @param response - * {@link HttpServletResponse} - * @param file - * Set of {@link AnalysisOutputFile} + * @param response {@link HttpServletResponse} + * @param file Set of {@link AnalysisOutputFile} + * @param iridaFileStorageService file storage implementation */ - public static void createSingleFileResponse(HttpServletResponse response, AnalysisOutputFile file) { + public static void createSingleFileResponse(HttpServletResponse response, AnalysisOutputFile file, IridaFileStorageService iridaFileStorageService) { String fileName = file.getLabel(); - FileUtilities.createSingleFileResponse(response, file, fileName); + FileUtilities.createSingleFileResponse(response, file, fileName, iridaFileStorageService); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java index f61f457008c..930db9bdd87 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java @@ -23,6 +23,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.AnalysisType; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.analysis.auditing.AnalysisAudit; import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysesListRequest; @@ -52,12 +53,13 @@ public class AnalysesTableAjaxController { private MessageSource messageSource; private UpdateAnalysisSubmissionPermission updateAnalysisSubmissionPermission; private AnalysisAudit analysisAudit; + private IridaFileStorageService iridaFileStorageService; @Autowired public AnalysesTableAjaxController(AnalysisSubmissionService analysisSubmissionService, AnalysisTypesService analysisTypesService, ProjectService projectService, IridaWorkflowsService iridaWorkflowsService, MessageSource messageSource, - UpdateAnalysisSubmissionPermission updateAnalysisSubmissionPermission, AnalysisAudit analysisAudit) { + UpdateAnalysisSubmissionPermission updateAnalysisSubmissionPermission, AnalysisAudit analysisAudit, IridaFileStorageService iridaFileStorageService) { this.analysisSubmissionService = analysisSubmissionService; this.analysisTypesService = analysisTypesService; this.projectService = projectService; @@ -65,6 +67,7 @@ public AnalysesTableAjaxController(AnalysisSubmissionService analysisSubmissionS this.messageSource = messageSource; this.updateAnalysisSubmissionPermission = updateAnalysisSubmissionPermission; this.analysisAudit = analysisAudit; + this.iridaFileStorageService = iridaFileStorageService; } /** @@ -248,7 +251,7 @@ public void downloadAnalysis(@PathVariable Long id, HttpServletResponse response Analysis analysis = analysisSubmission.getAnalysis(); Set files = analysis.getAnalysisOutputFiles(); - FileUtilities.createAnalysisOutputFileZippedResponse(response, analysisSubmission.getName(), files); + FileUtilities.createAnalysisOutputFileZippedResponse(response, analysisSubmission.getName(), files, iridaFileStorageService); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 7733e58428d..0c88598b99a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -9,6 +9,7 @@ import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -803,7 +804,7 @@ public void getAjaxDownloadAnalysisSubmission(@PathVariable Long analysisSubmiss Analysis analysis = analysisSubmission.getAnalysis(); Set files = analysis.getAnalysisOutputFiles(); - FileUtilities.createAnalysisOutputFileZippedResponse(response, analysisSubmission.getName(), files); + FileUtilities.createAnalysisOutputFileZippedResponse(response, analysisSubmission.getName(), files, iridaFileStorageService); } /** @@ -831,7 +832,7 @@ public Map prepareDownload(@RequestBody List ou @RequestMapping(value = "/download/selection", produces = MediaType.APPLICATION_JSON_VALUE) public void downloadSelection(@RequestParam(required = false, defaultValue = "analysis-output-files-batch-download") String filename, HttpServletResponse response) { Map files = analysisOutputFileDownloadManager.getSelection(); - FileUtilities.createBatchAnalysisOutputFileZippedResponse(response, filename, files); + FileUtilities.createBatchAnalysisOutputFileZippedResponse(response, filename, files, iridaFileStorageService); } /** @@ -859,9 +860,9 @@ public void getAjaxDownloadAnalysisSubmissionIndividualFile(@PathVariable Long a } if (!Strings.isNullOrEmpty(filename)) { - FileUtilities.createSingleFileResponse(response, optFile.get(), filename); + FileUtilities.createSingleFileResponse(response, optFile.get(), filename, iridaFileStorageService); } else { - FileUtilities.createSingleFileResponse(response, optFile.get()); + FileUtilities.createSingleFileResponse(response, optFile.get(), iridaFileStorageService); } } @@ -880,7 +881,7 @@ public Map getNewickForAnalysis(@PathVariable Long submissionId) AnalysisSubmission submission = analysisSubmissionService.read(submissionId); Analysis analysis = submission.getAnalysis(); AnalysisOutputFile file = analysis.getAnalysisOutputFile(treeFileKey); - List lines = Files.readAllLines(file.getFile()); + List lines = IOUtils.readLines(iridaFileStorageService.getFileInputStream(file.getFile())); return ImmutableMap.of("newick", lines.get(0)); } @@ -1047,7 +1048,7 @@ public AnalysisTreeResponse getNewickTree(@PathVariable Long submissionId, Local new Object[] {}, locale); } else { try { - List lines = Files.readAllLines(file.getFile()); + List lines = IOUtils.readLines(iridaFileStorageService.getFileInputStream(file.getFile())); if (lines.size() > 0) { tree = lines.get(0); From 9de15230df0491a834e86ea88198070c0f2a6404 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 4 May 2020 18:48:48 -0500 Subject: [PATCH 057/655] Updated getTemporaryFile methods for cloud storage to create a temp folder in the location provided --- .../services/IridaApiServicesConfig.java | 7 ++- .../IridaFileStorageAwsServiceImpl.java | 45 +++++++++++++------ .../IridaFileStorageAzureServiceImpl.java | 31 ++++++++++--- .../irida/config/filesystem.properties | 2 + 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 462bd2f8b68..934f42196f4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -149,6 +149,9 @@ public class IridaApiServicesConfig { @Value("${aws.secret.key:#{null}}") private String awsSecretKey; + @Value("${galaxy.cloud.storage.temporary.directory}") + private String cloudStorageTemporaryDirectory; + @Autowired private IridaPluginConfig.IridaPluginList pipelinePlugins; @@ -310,9 +313,9 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageService") public IridaFileStorageService iridaFileStorageService() { if(storageType.equalsIgnoreCase("azure")) { - return new IridaFileStorageAzureServiceImpl(connectionStr, containerName); + return new IridaFileStorageAzureServiceImpl(connectionStr, containerName, cloudStorageTemporaryDirectory); } else if (storageType.equalsIgnoreCase("aws")) { - return new IridaFileStorageAwsServiceImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); + return new IridaFileStorageAwsServiceImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey, cloudStorageTemporaryDirectory); } else { return new IridaFileStorageLocalServiceImpl(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 2003d980481..9957ba18626 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -2,16 +2,23 @@ import java.io.*; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Date; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.zip.GZIPInputStream; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; @@ -34,16 +41,22 @@ public class IridaFileStorageAwsServiceImpl implements IridaFileStorageService{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsServiceImpl.class); + @Value("${galaxy.tempfile.directory.permissions}") + private String filePermissions; + private String bucketName; private BasicAWSCredentials awsCreds; private AmazonS3 s3; + private String tempDir; + @Autowired - public IridaFileStorageAwsServiceImpl(String bucketName, String bucketRegion, String accessKey, String secretKey){ + public IridaFileStorageAwsServiceImpl(String bucketName, String bucketRegion, String accessKey, String secretKey, String cloudStorageTemporaryDirectory){ this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); this.s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion)) .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); this.bucketName = bucketName; + this.tempDir = cloudStorageTemporaryDirectory; } /** @@ -54,9 +67,6 @@ public File getTemporaryFile(Path file) { File fileToProcess = null; try { - - String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); - // Since the file system is virtual the full file path is the file name. // We split it on "/" and get the last token which is the actual file name. String [] nameTokens = file.toAbsolutePath().toString().split("/"); @@ -65,19 +75,28 @@ public File getTemporaryFile(Path file) { S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); - fileToProcess = new File(tmpDir+fileName); + FileAttribute> fileAttributes = PosixFilePermissions + .asFileAttribute(PosixFilePermissions.fromString(filePermissions)); + + Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "objectStoreTemp", fileAttributes); + Path target = targetDirectory.resolve(fileName); // Stream the file from the bucket into a local file - FileOutputStream fileOutputStream = new FileOutputStream(fileToProcess); +// FileOutputStream fileOutputStream = new FileOutputStream(fileToProcess); logger.trace("Downloading s3 object to: " + fileToProcess.getAbsolutePath()); - byte[] read_buf = new byte[1024]; - int read_len = 0; - while ((read_len = s3ObjectInputStream.read(read_buf)) > 0) { - fileOutputStream.write(read_buf, 0, read_len); - } + File targetFile = new File(target.toAbsolutePath().toString()); + FileUtils.copyInputStreamToFile(s3ObjectInputStream, targetFile); + + fileToProcess = targetFile; + +// byte[] read_buf = new byte[1024]; +// int read_len = 0; +// while ((read_len = s3ObjectInputStream.read(read_buf)) > 0) { +// fileOutputStream.write(read_buf, 0, read_len); +// } s3ObjectInputStream.close(); - fileOutputStream.close(); +// fileOutputStream.close(); s3Object.close(); } catch (AmazonServiceException e) { logger.error(e.getErrorMessage()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 308ed7c72fa..512813085a3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -7,15 +7,21 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Date; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.zip.GZIPInputStream; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; @@ -36,15 +42,21 @@ public class IridaFileStorageAzureServiceImpl implements IridaFileStorageService { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureServiceImpl.class); + @Value("${galaxy.tempfile.directory.permissions}") + private String filePermissions; + private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient ; private BlobClient blobClient; + private String tempDir; + @Autowired - public IridaFileStorageAzureServiceImpl(String connectionStr, String containerName){ + public IridaFileStorageAzureServiceImpl(String connectionStr, String containerName, String cloudStorageTemporaryDirectory){ this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + this.tempDir = cloudStorageTemporaryDirectory; } /** @@ -64,15 +76,22 @@ public File getTemporaryFile(Path file) { String [] blobNameTokens = blobClient.getBlobName().split("/"); String fileName = blobNameTokens[blobNameTokens.length-1]; - Path targetDirectory = Files.createTempDirectory(null); + FileAttribute> fileAttributes = PosixFilePermissions + .asFileAttribute(PosixFilePermissions.fromString(filePermissions)); + + Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "objectStoreTemp", fileAttributes); Path target = targetDirectory.resolve(fileName); - blobClient.downloadToFile(target.toAbsolutePath().toString()); - fileToProcess = new File(target.toAbsolutePath().toString()); + InputStream initialStream = blobClient.openInputStream(); + File targetFile = new File(target.toAbsolutePath().toString()); + FileUtils.copyInputStreamToFile(initialStream, targetFile); + + + fileToProcess = targetFile; } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); } catch (IOException e) { - logger.trace("Couldn't create temp directory [" + e + "]"); + logger.debug(e.getMessage()); } return fileToProcess; diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties index 1b8a047b91f..436e1b0d588 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties @@ -13,3 +13,5 @@ file.processing.process=true file.upload.max_size=21474836480 irida.storage.type=local + +galaxy.tempfile.directory.permissions=rwxrwxrwx \ No newline at end of file From b2be307e357fa29ead93226bd57bada3ad03838b Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 5 May 2020 11:14:35 -0500 Subject: [PATCH 058/655] Removed code not required. Changed cloud temp file directory prefix --- .../filesystem/IridaFileStorageAwsServiceImpl.java | 14 ++------------ .../IridaFileStorageAzureServiceImpl.java | 3 +-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java index 9957ba18626..e10ba253a03 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsServiceImpl.java @@ -78,25 +78,15 @@ public File getTemporaryFile(Path file) { FileAttribute> fileAttributes = PosixFilePermissions .asFileAttribute(PosixFilePermissions.fromString(filePermissions)); - Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "objectStoreTemp", fileAttributes); + Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "aws-temp-file-", fileAttributes); Path target = targetDirectory.resolve(fileName); - // Stream the file from the bucket into a local file -// FileOutputStream fileOutputStream = new FileOutputStream(fileToProcess); - logger.trace("Downloading s3 object to: " + fileToProcess.getAbsolutePath()); - + // Copy the the file from the bucket into a local file File targetFile = new File(target.toAbsolutePath().toString()); FileUtils.copyInputStreamToFile(s3ObjectInputStream, targetFile); - fileToProcess = targetFile; -// byte[] read_buf = new byte[1024]; -// int read_len = 0; -// while ((read_len = s3ObjectInputStream.read(read_buf)) > 0) { -// fileOutputStream.write(read_buf, 0, read_len); -// } s3ObjectInputStream.close(); -// fileOutputStream.close(); s3Object.close(); } catch (AmazonServiceException e) { logger.error(e.getErrorMessage()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java index 512813085a3..777f050861c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureServiceImpl.java @@ -79,14 +79,13 @@ public File getTemporaryFile(Path file) { FileAttribute> fileAttributes = PosixFilePermissions .asFileAttribute(PosixFilePermissions.fromString(filePermissions)); - Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "objectStoreTemp", fileAttributes); + Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "azure-temp-file-", fileAttributes); Path target = targetDirectory.resolve(fileName); InputStream initialStream = blobClient.openInputStream(); File targetFile = new File(target.toAbsolutePath().toString()); FileUtils.copyInputStreamToFile(initialStream, targetFile); - fileToProcess = targetFile; } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); From 90cfe53775e71c23153b9fde660abf5751052bee Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 20 May 2020 16:00:46 -0500 Subject: [PATCH 059/655] Updated controller to fix failing tests --- .../irida/ria/web/analysis/AnalysesTableAjaxController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java index f61f457008c..e0a5a49cac2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java @@ -74,6 +74,7 @@ public AnalysesTableAjaxController(AnalysisSubmissionService analysisSubmissionS * @return {@link List} of {@link AnalysisState} */ @RequestMapping("/states") + @ResponseBody public List getAnalysisStates(Locale locale) { List states = Arrays.asList(AnalysisState.values()); return states.stream() @@ -89,6 +90,7 @@ public List getAnalysisStates(Locale locale) { * @return an internationalized list of analysis type names */ @RequestMapping("/types") + @ResponseBody public List getWorkflowTypes(Locale locale) { Set types = iridaWorkflowsService.getRegisteredWorkflowTypes(); return types.stream() From 8477bb95ad7d2dd6a2e12591e61c4870bba42f7f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 17 Jun 2020 12:48:35 -0500 Subject: [PATCH 060/655] Renamed IridaFileStorageService to IridaFileStorageUtility. Moved setIridaFileStorageUtility method into VersionedFields to remove unnecessary reflection. Removed methods not used. Added file methods to sequencefile so we can call them directly rather than autowiring in the iridafilestorageutility into files where it doesn't need to be --- .../services/IridaApiServicesConfig.java | 10 +++---- .../irida/model/VersionedFileFields.java | 8 +++++ .../irida/model/assembly/GenomeAssembly.java | 10 +++---- .../irida/model/project/ReferenceFile.java | 11 ++++++- .../irida/model/sequenceFile/Fast5Object.java | 2 +- .../model/sequenceFile/SequenceFile.java | 29 ++++++++++++++----- .../workflow/analysis/AnalysisOutputFile.java | 11 ++++++- .../SequencingObjectConcatenatorFactory.java | 16 +++++----- .../impl/SequenceFilePairConcatenator.java | 14 ++++----- .../SingleEndSequenceFileConcatenator.java | 12 ++++---- .../impl/ChecksumFileProcessor.java | 7 ++--- .../impl/DefaultFileProcessingChain.java | 11 +++---- .../processing/impl/FastqcFileProcessor.java | 12 ++++---- .../processing/impl/GzipFileProcessor.java | 13 +++------ .../AnalysisOutputFileRepositoryImpl.java | 6 ++-- .../GenomeAssemblyRepositoryImpl.java | 6 ++-- .../listeners/IridaFileStorageListener.java | 29 +++++-------------- .../FilesystemSupplementedRepositoryImpl.java | 8 ++--- ... => IridaFileStorageLocalUtilityImpl.java} | 27 ++--------------- ...vice.java => IridaFileStorageUtility.java} | 20 +------------ .../ReferenceFileRepositoryImpl.java | 6 ++-- .../SequenceFileRepositoryImpl.java | 6 ++-- .../ProjectReferenceFileController.java | 10 +++---- .../impl/SequencingObjectServiceImpl.java | 10 +++---- .../webapp/pages/samples/sample_files.html | 2 +- ...quencingObjectConcatenatorFactoryTest.java | 20 ++++++------- .../SequenceFilePairConcatenatorTest.java | 10 +++---- ...SingleEndSequenceFileConcatenatorTest.java | 10 +++---- .../impl/unit/ChecksumFileProcessorTest.java | 10 +++---- .../unit/DefaultFileProcessingChainTest.java | 16 ++++------ .../impl/unit/FastqcFileProcessorTest.java | 10 +++---- .../impl/unit/GzipFileProcessorTest.java | 8 ++--- .../SequenceFileRepositoryImplTest.java | 6 ++-- .../unit/SequencingObjectServiceTest.java | 10 +++---- 34 files changed, 182 insertions(+), 214 deletions(-) rename src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/{IridaFileStorageLocalServiceImpl.java => IridaFileStorageLocalUtilityImpl.java} (93%) rename src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/{IridaFileStorageService.java => IridaFileStorageUtility.java} (89%) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 9667548da4f..be1ed9c985a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -286,14 +286,14 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { return null; } - @Bean(name = "iridaFileStorageService") - public IridaFileStorageService iridaFileStorageService() { - return new IridaFileStorageLocalServiceImpl(); + @Bean(name = "iridaFileStorageUtility") + public IridaFileStorageUtility iridaFileStorageUtility() { + return new IridaFileStorageLocalUtilityImpl(); } @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, GzipFileProcessor gzipFileProcessor, + QCEntryRepository qcRepository, GzipFileProcessor gzipFileProcessor, FastqcFileProcessor fastQcFileProcessor, ChecksumFileProcessor checksumProcessor, CoverageFileProcessor coverageProcessor, AutomatedAnalysisFileProcessor automatedAnalysisFileProcessor) { @@ -307,7 +307,7 @@ public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequenc fileProcessors.remove(gzipFileProcessor); } - return new DefaultFileProcessingChain(sequencingObjectRepository, qcRepository, iridaFileStorageService, fileProcessors); + return new DefaultFileProcessingChain(sequencingObjectRepository, qcRepository, fileProcessors); } @Bean(name = "fileProcessingChainExecutor") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java index dd4c4ce0b25..c492fcad7a0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java @@ -4,6 +4,8 @@ import javax.persistence.Version; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; + /** * An instance of a class may have a property with {@link Version} or may have * an internally managed version representation. This interface exposes that @@ -26,4 +28,10 @@ public interface VersionedFileFields { * Internally modify the file revision number to something new. */ public void incrementFileRevisionNumber(); + + /** + * + * @param iridaFileStorageUtility instance of iridaFileStorageUtility to provide to entity + */ + public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 81ffa146d4c..56c9bfb1f65 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -21,7 +21,7 @@ import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; import ca.corefacility.bioinformatics.irida.model.joins.impl.SampleGenomeAssemblyJoin; import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Lists; @@ -38,7 +38,7 @@ public abstract class GenomeAssembly extends IridaResourceSupport implements IridaThing, IridaSequenceFile, VersionedFileFields { private static final Logger logger = LoggerFactory.getLogger(GenomeAssembly.class); - private static IridaFileStorageService iridaFileStorageService; + private static IridaFileStorageUtility iridaFileStorageUtility; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -110,7 +110,7 @@ public void addSampleGenomeAssemblyJoin(SampleGenomeAssemblyJoin join) { @JsonIgnore public String getFileSize() { String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageService.getFileSize(getFile()), true); + size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageUtility.getFileSize(getFile()), true); return size; } @@ -147,7 +147,7 @@ public boolean equals(Object obj) { return Objects.equals(this.id, other.id) && Objects.equals(this.createdDate, other.createdDate); } - public void setIridaFileStorageService(IridaFileStorageService iridaFileStorageService) { - this.iridaFileStorageService = iridaFileStorageService; + public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index 8ca4aee9315..505940521fd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -25,7 +25,9 @@ import ca.corefacility.bioinformatics.irida.model.MutableIridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * A reference file to be associated with a {@link Project}. @@ -35,9 +37,11 @@ @Entity @Table(name = "reference_file") @Audited -@EntityListeners({AuditingEntityListener.class, RelativePathTranslatorListener.class}) +@EntityListeners({AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageListener.class}) public class ReferenceFile implements VersionedFileFields, MutableIridaThing { + private static IridaFileStorageUtility iridaFileStorageUtility; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -144,4 +148,9 @@ public Long getFileLength() { public void setFileLength(Long fileLength) { this.fileLength = fileLength; } + + @Override + public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 48c4ca7f849..4475dc916dc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -107,7 +107,7 @@ private Fast5Type setType(SequenceFile sequenceFile) { try { String extension = FilenameUtils.getExtension(file.toString()); - boolean gzipped = FileUtils.isGzipped(file); + boolean gzipped = sequenceFile.isGzipped(); if (gzipped) { type = Fast5Object.Fast5Type.ZIPPED; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 70787389319..b7683795fa1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -1,5 +1,8 @@ package ca.corefacility.bioinformatics.irida.model.sequenceFile; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.Date; import java.util.HashMap; @@ -28,7 +31,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.fasterxml.jackson.annotation.JsonAnyGetter; @@ -48,7 +51,7 @@ public class SequenceFile extends IridaResourceSupport implements MutableIridaTh private static final Logger logger = LoggerFactory.getLogger(SequenceFile.class); - private static IridaFileStorageService iridaFileStorageService; + private static IridaFileStorageUtility iridaFileStorageUtility; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -121,7 +124,7 @@ public String getLabel() { @JsonIgnore public String getFileSize() { String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageService.getFileSize(getFile()), true); + size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageUtility.getFileSize(getFile()), true); return size; } @@ -298,12 +301,24 @@ public void setUploadSha256(String uploadSha256) { this.uploadSha256 = uploadSha256; } - public IridaFileStorageService getIridaFileStorageService() { - return iridaFileStorageService; + @Override + public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; + } + + public InputStream getFileInputStream() { + return iridaFileStorageUtility.getFileInputStream(getFile()); } - public void setIridaFileStorageService(IridaFileStorageService iridaFileStorageService) { - this.iridaFileStorageService = iridaFileStorageService; + public boolean isGzipped() throws Exception { + return iridaFileStorageUtility.isGzipped(getFile()); } + public File getTemporaryFile() { + return iridaFileStorageUtility.getTemporaryFile(getFile()); + } + + public boolean fileExists() { + return iridaFileStorageUtility.fileExists(getFile()); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 8888163ffdd..0688915c0ba 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -26,8 +26,10 @@ import ca.corefacility.bioinformatics.irida.model.IridaResourceSupport; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; +import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Store file references to files produced by a workflow execution that we @@ -37,9 +39,11 @@ */ @Entity @Table(name = "analysis_output_file") -@EntityListeners(RelativePathTranslatorListener.class) +@EntityListeners({ RelativePathTranslatorListener.class, IridaFileStorageListener.class }) public class AnalysisOutputFile extends IridaResourceSupport implements IridaThing, VersionedFileFields { + private static IridaFileStorageUtility iridaFileStorageUtility; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private final Long id; @@ -188,4 +192,9 @@ public byte[] getBytesForFile() throws IOException { byte[] bytes = Files.readAllBytes(getFile()); return bytes; } + + @Override + public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java index b0e98483f6a..b6d155081f8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java @@ -9,7 +9,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Factory class for returning an instance of @@ -22,17 +22,17 @@ public class SequencingObjectConcatenatorFactory { * * @param type the class to get a concatenator for * @param The type this concatenator should act on - * @param iridaFileStorageService The file storage component + * @param iridaFileStorageUtility The file storage component * @return the new {@link SequencingObjectConcatenator} */ @SuppressWarnings("unchecked") - public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageService iridaFileStorageService) { + public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageUtility iridaFileStorageUtility) { // return the concatenator for the class if (type.equals(SingleEndSequenceFile.class)) { - return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageService); + return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageUtility); } else if (type.equals(SequenceFilePair.class)) { - return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageService); + return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageUtility); } else { throw new IllegalArgumentException("No concatenator exists for type " + type); } @@ -43,11 +43,11 @@ public static SequencingObjectConcatenator getCo * {@link SequencingObject}s * * @param objects the {@link SequencingObject}s to get the concatenator for - * @param iridaFileStorageService The file storage component + * @param iridaFileStorageUtility The file storage component * @return the new {@link SequencingObjectConcatenator} */ public static SequencingObjectConcatenator getConcatenator( - Collection objects, IridaFileStorageService iridaFileStorageService) { + Collection objects, IridaFileStorageUtility iridaFileStorageUtility) { // get all the classes for the objects Set types = objects.stream().map(Object::getClass).collect(Collectors.toSet()); @@ -61,6 +61,6 @@ public static SequencingObjectConcatenator getConcat @SuppressWarnings("unchecked") Class type = (Class) types.iterator().next(); - return getConcatenator(type, iridaFileStorageService); + return getConcatenator(type, iridaFileStorageUtility); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 043b9896186..3c4b6091f02 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import java.io.IOException; import java.nio.file.Files; @@ -21,11 +21,11 @@ @Component public class SequenceFilePairConcatenator extends SequencingObjectConcatenator { - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public SequenceFilePairConcatenator(IridaFileStorageService iridaFileStorageService) { - this.iridaFileStorageService = iridaFileStorageService; + public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -35,7 +35,7 @@ public SequenceFilePairConcatenator(IridaFileStorageService iridaFileStorageServ public SequenceFilePair concatenateFiles(List toConcatenate, String filename) throws ConcatenateException { - String extension = iridaFileStorageService.getFileExtension(toConcatenate); + String extension = iridaFileStorageUtility.getFileExtension(toConcatenate); // create the filenames with F/R for the forward and reverse files String forwardName = filename + "_R1." + extension; @@ -65,8 +65,8 @@ public SequenceFilePair concatenateFiles(List toConc SequenceFile forwardSequenceFile = pair.getForwardSequenceFile(); SequenceFile reverseSequenceFile = pair.getReverseSequenceFile(); - iridaFileStorageService.appendToFile(forwardFile, forwardSequenceFile); - iridaFileStorageService.appendToFile(reverseFile, reverseSequenceFile); + iridaFileStorageUtility.appendToFile(forwardFile, forwardSequenceFile); + iridaFileStorageUtility.appendToFile(reverseFile, reverseSequenceFile); } // create new SequenceFiles diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index debfd2a54a1..f992b7f2f3b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import java.io.IOException; import java.nio.file.Files; @@ -21,11 +21,11 @@ @Component public class SingleEndSequenceFileConcatenator extends SequencingObjectConcatenator { - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public SingleEndSequenceFileConcatenator(IridaFileStorageService iridaFileStorageService) { - this.iridaFileStorageService = iridaFileStorageService; + public SingleEndSequenceFileConcatenator(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -36,7 +36,7 @@ public SingleEndSequenceFile concatenateFiles(List t throws ConcatenateException { Path tempFile; - String extension = iridaFileStorageService.getFileExtension(toConcatenate); + String extension = iridaFileStorageUtility.getFileExtension(toConcatenate); // create the filename with extension filename = filename + "." + extension; @@ -58,7 +58,7 @@ public SingleEndSequenceFile concatenateFiles(List t SequenceFile forwardSequenceFile = single.getSequenceFile(); - iridaFileStorageService.appendToFile(tempFile, forwardSequenceFile); + iridaFileStorageUtility.appendToFile(tempFile, forwardSequenceFile); } // create the new sequencefile and object diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index 15cbc9b4cad..345c6947ddf 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -14,7 +14,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -26,12 +25,10 @@ public class ChecksumFileProcessor implements FileProcessor { private static final Logger logger = LoggerFactory.getLogger(ChecksumFileProcessor.class); private SequenceFileRepository fileRepository; - private IridaFileStorageService iridaFileStorageService; @Autowired - public ChecksumFileProcessor(SequenceFileRepository fileRepository, IridaFileStorageService iridaFileStorageService) { + public ChecksumFileProcessor(SequenceFileRepository fileRepository) { this.fileRepository = fileRepository; - this.iridaFileStorageService = iridaFileStorageService; } /** @@ -50,7 +47,7 @@ public void process(SequencingObject sequencingObject) { for (SequenceFile file : files) { - try (InputStream is = iridaFileStorageService.getFileInputStream(file.getFile())) { + try (InputStream is = file.getFileInputStream()) { String shaDigest = DigestUtils.sha256Hex(is); logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java index dce3e1d04fc..7b1ba9e98fe 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java @@ -12,7 +12,6 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessingChain; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -36,19 +35,17 @@ public class DefaultFileProcessingChain implements FileProcessingChain { private final SequencingObjectRepository sequencingObjectRepository; private QCEntryRepository qcRepository; - private IridaFileStorageService iridaFileStorageService; public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, FileProcessor... fileProcessors) { - this(sequencingObjectRepository, qcRepository, iridaFileStorageService, Arrays.asList(fileProcessors)); + QCEntryRepository qcRepository, FileProcessor... fileProcessors) { + this(sequencingObjectRepository, qcRepository, Arrays.asList(fileProcessors)); } public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, IridaFileStorageService iridaFileStorageService, List fileProcessors) { + QCEntryRepository qcRepository, List fileProcessors) { this.fileProcessors = fileProcessors; this.sequencingObjectRepository = sequencingObjectRepository; this.qcRepository = qcRepository; - this.iridaFileStorageService = iridaFileStorageService; } /** @@ -178,7 +175,7 @@ private SequencingObject getSettledSequencingObject(Long sequencingObjectId) thr if(sequencingObject.isPresent()) { Set files = sequencingObject.get().getFiles(); filesNotSettled = files.stream().anyMatch(f -> { - return !iridaFileStorageService.fileExists(f.getFile()); + return !f.fileExists(); }); } } while (filesNotSettled); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 9f866902084..af97b2023dd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -10,7 +10,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import org.slf4j.Logger; @@ -54,7 +54,7 @@ public class FastqcFileProcessor implements FileProcessor { private final SequenceFileRepository sequenceFileRepository; private final AnalysisOutputFileRepository outputFileRepository; private final MessageSource messageSource; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; /** * Create a new {@link FastqcFileProcessor} @@ -63,15 +63,15 @@ public class FastqcFileProcessor implements FileProcessor { * analysis). * @param sequenceFileRepository Repository for storing sequence files * @param outputFileRepository Repository for storing analysis output files - * @param iridaFileStorageService The irida file storage service + * @param iridaFileStorageUtility The irida file storage service */ @Autowired public FastqcFileProcessor(final MessageSource messageSource, final SequenceFileRepository sequenceFileRepository, - AnalysisOutputFileRepository outputFileRepository, IridaFileStorageService iridaFileStorageService) { + AnalysisOutputFileRepository outputFileRepository, IridaFileStorageUtility iridaFileStorageUtility) { this.messageSource = messageSource; this.sequenceFileRepository = sequenceFileRepository; this.outputFileRepository = outputFileRepository; - this.iridaFileStorageService = iridaFileStorageService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } @Override @@ -97,7 +97,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx LocaleContextHolder.getLocale())); try { uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - iridaFileStorageService.getTemporaryFile(fileToProcess)); + sequenceFile.getTemporaryFile()); BasicStats basicStats = new BasicStats(); PerBaseQualityScores pbqs = new PerBaseQualityScores(); PerSequenceQualityScores psqs = new PerSequenceQualityScores(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 664d0da81e6..596103fb834 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -17,7 +17,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -39,19 +38,15 @@ public class GzipFileProcessor implements FileProcessor { private boolean disableFileProcessor = false; private boolean removeCompressedFile; - private IridaFileStorageService iridaFileStorageService; - @Autowired - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, IridaFileStorageService iridaFileStorageService) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository) { this.sequenceFileRepository = sequenceFileRepository; removeCompressedFile = false; - this.iridaFileStorageService = iridaFileStorageService; } - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles, IridaFileStorageService iridaFileStorageService) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles) { this.sequenceFileRepository = sequenceFileRepository; this.removeCompressedFile = removeCompressedFiles; - this.iridaFileStorageService = iridaFileStorageService; } /** @@ -115,10 +110,10 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); - if (iridaFileStorageService.isGzipped(file)) { + if (sequenceFile.isGzipped()) { file = addExtensionToFilename(file, GZIP_EXTENSION); - try (GZIPInputStream zippedInputStream = new GZIPInputStream(iridaFileStorageService.getFileInputStream(file))) { + try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { logger.trace("Handling gzip compressed file."); Path targetDirectory = Files.createTempDirectory(null); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java index 17aea89dfda..6f7e5bb4dfc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java @@ -11,7 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Custom implementation of {@link FilesystemSupplementedRepositoryImpl} for @@ -24,8 +24,8 @@ public class AnalysisOutputFileRepositoryImpl extends FilesystemSupplementedRepo @Autowired public AnalysisOutputFileRepositoryImpl(EntityManager entityManager, - @Qualifier("outputFileBaseDirectory") Path baseDirectory, IridaFileStorageService iridaFileStorageService) { - super(entityManager, baseDirectory, iridaFileStorageService); + @Qualifier("outputFileBaseDirectory") Path baseDirectory, IridaFileStorageUtility iridaFileStorageUtility) { + super(entityManager, baseDirectory, iridaFileStorageUtility); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java index 36cdd33342c..f26c0062862 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java @@ -11,7 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; import ca.corefacility.bioinformatics.irida.model.assembly.UploadedAssembly; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * A {@link FilesystemSupplementedRepositoryImpl} implementation for {@link GenomeAssembly} @@ -21,8 +21,8 @@ public class GenomeAssemblyRepositoryImpl extends FilesystemSupplementedReposito @Autowired public GenomeAssemblyRepositoryImpl(EntityManager entityManager, - @Qualifier("assemblyFileBaseDirectory") Path baseDirectory, IridaFileStorageService iridaFileStorageService) { - super(entityManager, baseDirectory, iridaFileStorageService); + @Qualifier("assemblyFileBaseDirectory") Path baseDirectory, IridaFileStorageUtility iridaFileStorageUtility) { + super(entityManager, baseDirectory, iridaFileStorageUtility); } @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java index 81c773c426e..6ed20b73e33 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java @@ -1,8 +1,5 @@ package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - import javax.persistence.PostLoad; import org.slf4j.Logger; @@ -12,39 +9,27 @@ import org.springframework.web.context.support.SpringBeanAutowiringSupport; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** - * Component implementation to run on an entity after it is has been accessed from the db. + * Component implementation to run on a versioned entity after it is has been accessed from the db. */ @Component public class IridaFileStorageListener { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageListener.class); @Autowired - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; /** - * After the entity is loaded this method will provide - * the entity access to the iridaFileStorageService + * After the versioned entity is loaded this method will provide + * the entity access to the iridaFileStorageUtility * - * @param fileSystemEntity The entity to provide the iridaFileStorageService to + * @param fileSystemEntity The versioned entity to provide the iridaFileStorageUtility to */ @PostLoad public void afterEntityLoad(final VersionedFileFields fileSystemEntity) { - try { SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); - // Use reflection to get the setIridaFileStorageService method, make it accessible, and invoke it - Method iridaFileStorageServiceSetter = fileSystemEntity.getClass() - .getMethod("setIridaFileStorageService", IridaFileStorageService.class); - iridaFileStorageServiceSetter.setAccessible(true); - iridaFileStorageServiceSetter.invoke(fileSystemEntity, iridaFileStorageService); - } catch (NoSuchMethodException e) { - logger.error("The specified method does not exist. " + e.getMessage()); - } catch (IllegalAccessException e) { - logger.error(e.getMessage()); - } catch (InvocationTargetException e) { - logger.error(e.getMessage()); - } + fileSystemEntity.setIridaFileStorageUtility(iridaFileStorageUtility); } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java index c1181734144..44218827f91 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java @@ -40,12 +40,12 @@ public abstract class FilesystemSupplementedRepositoryImpl VALID_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** @@ -61,24 +61,6 @@ public interface IridaFileStorageService { */ public String getFileName(Path file); - /** - * Deletes the file from (azure, aws, or local disk) - * - */ - public void deleteFile(); - - /** - * Download the file from (azure, aws, or local disk) - * - */ - public void downloadFile(); - - /** - * Downloads all the files of type `analysis-output` from - * (azure, aws, or local disk) - * - */ - public void downloadFiles(); /** * Checks if file exists diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/referencefile/ReferenceFileRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/referencefile/ReferenceFileRepositoryImpl.java index f50d67545fe..bd5c6307558 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/referencefile/ReferenceFileRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/referencefile/ReferenceFileRepositoryImpl.java @@ -11,7 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.util.SequenceFileUtilities; /** @@ -28,8 +28,8 @@ public class ReferenceFileRepositoryImpl extends FilesystemSupplementedRepositor @Autowired public ReferenceFileRepositoryImpl(final EntityManager entityManager, final SequenceFileUtilities sequenceFileUtilities, - final @Qualifier("referenceFileBaseDirectory") Path baseDirectory, IridaFileStorageService iridaFileStorageService) { - super(entityManager, baseDirectory, iridaFileStorageService); + final @Qualifier("referenceFileBaseDirectory") Path baseDirectory, IridaFileStorageUtility iridaFileStorageUtility) { + super(entityManager, baseDirectory, iridaFileStorageUtility); this.sequenceFileUtilities = sequenceFileUtilities; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/sequencefile/SequenceFileRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/sequencefile/SequenceFileRepositoryImpl.java index 72fef7fc5ad..d758e8a8fbc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/sequencefile/SequenceFileRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/sequencefile/SequenceFileRepositoryImpl.java @@ -11,7 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Custom implementation of {@link FilesystemSupplementedRepositoryImpl} for @@ -24,8 +24,8 @@ public class SequenceFileRepositoryImpl extends FilesystemSupplementedRepository @Autowired public SequenceFileRepositoryImpl(EntityManager entityManager, - @Qualifier("sequenceFileBaseDirectory") Path baseDirectory, IridaFileStorageService iridaFileStorageService) { - super(entityManager, baseDirectory, iridaFileStorageService); + @Qualifier("sequenceFileBaseDirectory") Path baseDirectory, IridaFileStorageUtility iridaFileStorageUtility) { + super(entityManager, baseDirectory, iridaFileStorageUtility); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java index 6b9017c89f3..b37a115a3ac 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/settings/ProjectReferenceFileController.java @@ -8,7 +8,7 @@ import java.util.Locale; import java.util.Map; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.projects.ProjectControllerUtils; import ca.corefacility.bioinformatics.irida.ria.web.projects.ProjectsController; import org.slf4j.Logger; @@ -42,16 +42,16 @@ public class ProjectReferenceFileController { private final ReferenceFileService referenceFileService; private final ProjectControllerUtils projectControllerUtils; private final MessageSource messageSource; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired public ProjectReferenceFileController(ProjectService projectService, ReferenceFileService referenceFileService, - ProjectControllerUtils projectControllerUtils, MessageSource messageSource, IridaFileStorageService iridaFileStorageService) { + ProjectControllerUtils projectControllerUtils, MessageSource messageSource, IridaFileStorageUtility iridaFileStorageUtility) { this.projectService = projectService; this.referenceFileService = referenceFileService; this.projectControllerUtils = projectControllerUtils; this.messageSource = messageSource; - this.iridaFileStorageService = iridaFileStorageService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -94,7 +94,7 @@ public String getProjectReferenceFilesPage(final Model model, final Principal pr map.put("label", file.getLabel()); map.put("createdDate", file.getCreatedDate()); Path path = file.getFile(); - map.put("size", iridaFileStorageService.getFileSize(path)); + map.put("size", iridaFileStorageUtility.getFileSize(path)); files.add(map); } return ImmutableMap.of("files", files); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index a118b7f6483..476bb1b713e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -12,7 +12,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenatorFactory; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -45,19 +45,19 @@ public class SequencingObjectServiceImpl extends CRUDServiceImpl to throws ConcatenateException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory - .getConcatenator(toJoin, iridaFileStorageService); + .getConcatenator(toJoin, iridaFileStorageUtility); SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); diff --git a/src/main/webapp/pages/samples/sample_files.html b/src/main/webapp/pages/samples/sample_files.html index 1778c2adbb9..2405a7cb24a 100644 --- a/src/main/webapp/pages/samples/sample_files.html +++ b/src/main/webapp/pages/samples/sample_files.html @@ -200,7 +200,7 @@

FAST5 Files

+ th:text="${file.getFileSize()}">
File Size
diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index 38e2a3a4517..c35ca9382b0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -5,8 +5,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.google.common.collect.Sets; @@ -23,49 +23,49 @@ */ public class SequencingObjectConcatenatorFactoryTest { - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); } @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class, iridaFileStorageService); + SingleEndSequenceFile.class, iridaFileStorageUtility); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class, iridaFileStorageService); + SequenceFilePair.class, iridaFileStorageUtility); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageService); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageUtility); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageService); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index f3131500852..b3ced361c23 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -3,8 +3,8 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.google.common.collect.Lists; import org.junit.Before; @@ -24,12 +24,12 @@ public class SequenceFilePairConcatenatorTest { + SEQUENCE + "\n+\n?????????").getBytes(); private SequenceFilePairConcatenator concat; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { - iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); - concat = new SequenceFilePairConcatenator(iridaFileStorageService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + concat = new SequenceFilePairConcatenator(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 2bdf1679d7a..b182ce6364e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -3,8 +3,8 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.google.common.collect.Lists; import org.junit.Before; @@ -24,12 +24,12 @@ public class SingleEndSequenceFileConcatenatorTest { + SEQUENCE + "\n+\n?????????").getBytes(); private SingleEndSequenceFileConcatenator concat; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { - iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); - concat = new SingleEndSequenceFileConcatenator(iridaFileStorageService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + concat = new SingleEndSequenceFileConcatenator(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index fa09ddeff67..d64354071f4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -17,8 +17,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.ChecksumFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; public class ChecksumFileProcessorTest { @@ -26,13 +26,13 @@ public class ChecksumFileProcessorTest { private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; private static final String CHECKSUM = "aeaa0755dc44b393ffe12f02e9bd42b0169b12ca9c15708085db6a4ac9110ee0"; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); - fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + fileProcessor = new ChecksumFileProcessor(sequenceFileRepository); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java index 405c07099af..3e76018557d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java @@ -26,8 +26,6 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.DefaultFileProcessingChain; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -40,7 +38,6 @@ public class DefaultFileProcessingChainTest { private SequencingObjectRepository objectRepository; private QCEntryRepository qcRepository; - private IridaFileStorageService iridaFileStorageService; private SequencingObject seqObject; private Long objectId = 1L; @@ -49,7 +46,6 @@ public class DefaultFileProcessingChainTest { public void setUp() { this.objectRepository = mock(SequencingObjectRepository.class); this.qcRepository = mock(QCEntryRepository.class); - this.iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); seqObject = new NoFileSequencingObject(); when(objectRepository.findById(objectId)).thenReturn(Optional.of(seqObject)); @@ -57,7 +53,7 @@ public void setUp() { @Test(expected = FileProcessorTimeoutException.class) public void testExceedsTimeout() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); fileProcessingChain.setTimeout(1); fileProcessingChain.setSleepDuration(0); @@ -66,7 +62,7 @@ public void testExceedsTimeout() throws FileProcessorTimeoutException { @Test public void testProcessEmptyChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); when(objectRepository.existsById(objectId)).thenReturn(true); fileProcessingChain.launchChain(objectId); @@ -74,7 +70,7 @@ public void testProcessEmptyChain() throws FileProcessorTimeoutException { @Test public void testFailWithContinueChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -88,7 +84,7 @@ public void testFailWithContinueChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFastFailProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -100,7 +96,7 @@ public void testFastFailProcessorChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFailOnProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -110,7 +106,7 @@ public void testFailOnProcessorChain() throws FileProcessorTimeoutException { @Test public void testFailWriteQCEntry() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageService, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 0d2a5e23700..d1ff7b2e4a7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -28,8 +28,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.FastqcFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -48,15 +48,15 @@ public class FastqcFileProcessorTest { private static final String FASTQ_FILE_CONTENTS = "@testread\n" + SEQUENCE + "\n+\n?????????\n@testread2\n" + SEQUENCE + "\n+\n?????????"; private static final String FASTA_FILE_CONTENTS = ">test read\n" + SEQUENCE; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { messageSource = mock(MessageSource.class); sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); - iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); - fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageUtility); } @Test(expected = FileProcessorException.class) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 240681616b5..8edbeea806c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -22,8 +22,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -36,13 +34,11 @@ public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; - private IridaFileStorageService iridaFileStorageService; @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageService); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE); } @Test(expected = FileProcessorException.class) @@ -64,7 +60,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageService); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java index 089b8f6503b..14d9c109fa0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/SequenceFileRepositoryImplTest.java @@ -33,14 +33,14 @@ public class SequenceFileRepositoryImplTest { private FilesystemSupplementedRepositoryImpl repository; private Path baseDirectory; private EntityManager entityManager; - private IridaFileStorageService iridaFileStorageService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() throws IOException { baseDirectory = Files.createTempDirectory(TEMP_FILE_PREFIX); entityManager = mock(EntityManager.class); - iridaFileStorageService = new IridaFileStorageLocalServiceImpl(); - repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + repository = new SequenceFileRepositoryImpl(entityManager, baseDirectory, iridaFileStorageUtility); } @After diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java index ff84de02946..48d9e5118fa 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java @@ -6,8 +6,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalServiceImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -32,7 +32,7 @@ public class SequencingObjectServiceTest { SampleSequencingObjectJoinRepository ssoRepository; SequenceConcatenationRepository concatenationRepository; Validator validator; - IridaFileStorageService iridaFileStorageService; + IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -40,9 +40,9 @@ public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); ssoRepository = mock(SampleSequencingObjectJoinRepository.class); concatenationRepository = mock(SequenceConcatenationRepository.class); - iridaFileStorageService = mock(IridaFileStorageLocalServiceImpl.class); + iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); service = new SequencingObjectServiceImpl(repository, sequenceFileRepository, ssoRepository, - concatenationRepository, validator, iridaFileStorageService); + concatenationRepository, validator, iridaFileStorageUtility); } @Test From 2f3e957f7a3a515362d5c8c107008236a1146e4d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 18 Jun 2020 15:34:48 -0500 Subject: [PATCH 061/655] Removed getTemporaryFile() from SequenceFile. Updated tests. --- .../model/sequenceFile/SequenceFile.java | 4 ---- .../processing/impl/FastqcFileProcessor.java | 2 +- .../listeners/IridaFileStorageListener.java | 1 + .../webapp/pages/samples/sample_files.html | 2 +- .../sequenceFile/unit/Fast5ObjectTest.java | 24 ++++++++++++++----- .../impl/unit/ChecksumFileProcessorTest.java | 3 ++- .../impl/unit/GzipFileProcessorTest.java | 14 ++++++++++- 7 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index b7683795fa1..429710412d0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -314,10 +314,6 @@ public boolean isGzipped() throws Exception { return iridaFileStorageUtility.isGzipped(getFile()); } - public File getTemporaryFile() { - return iridaFileStorageUtility.getTemporaryFile(getFile()); - } - public boolean fileExists() { return iridaFileStorageUtility.fileExists(getFile()); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index af97b2023dd..b8c20202d54 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -97,7 +97,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx LocaleContextHolder.getLocale())); try { uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - sequenceFile.getTemporaryFile()); + iridaFileStorageUtility.getTemporaryFile(fileToProcess)); BasicStats basicStats = new BasicStats(); PerBaseQualityScores pbqs = new PerBaseQualityScores(); PerSequenceQualityScores psqs = new PerSequenceQualityScores(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java index 6ed20b73e33..900c510bc8b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java @@ -29,6 +29,7 @@ public class IridaFileStorageListener { */ @PostLoad public void afterEntityLoad(final VersionedFileFields fileSystemEntity) { + // Required so that Spring's IoC container can handle the injection SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); fileSystemEntity.setIridaFileStorageUtility(iridaFileStorageUtility); } diff --git a/src/main/webapp/pages/samples/sample_files.html b/src/main/webapp/pages/samples/sample_files.html index 2405a7cb24a..1778c2adbb9 100644 --- a/src/main/webapp/pages/samples/sample_files.html +++ b/src/main/webapp/pages/samples/sample_files.html @@ -200,7 +200,7 @@

FAST5 Files

+ th:text="${file.getFile().getFileSize()}">
File Size
diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java index ef6aaa55806..b808033bbff 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java @@ -5,10 +5,13 @@ import java.nio.file.Path; import java.util.zip.GZIPOutputStream; +import org.junit.Before; import org.junit.Test; import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import static org.junit.Assert.assertEquals; @@ -17,12 +20,19 @@ */ public class Fast5ObjectTest { private static final String FILE_CONTENTS = "DATA CONTENTS"; + private IridaFileStorageUtility iridaFileStorageUtility; + + @Before + public void setUp() { + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + } @Test public void testCreateZippedFile() throws IOException { Path zipFile = createZipFile(); - - Fast5Object fast5Object = new Fast5Object(new SequenceFile(zipFile)); + SequenceFile sf = new SequenceFile(zipFile); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); + Fast5Object fast5Object = new Fast5Object(sf); assertEquals(Fast5Object.Fast5Type.ZIPPED, fast5Object.getFast5Type()); } @@ -30,8 +40,9 @@ public void testCreateZippedFile() throws IOException { @Test public void testCreateSingleFile() throws IOException { Path zipFile = createSingleFile(); - - Fast5Object fast5Object = new Fast5Object(new SequenceFile(zipFile)); + SequenceFile sf = new SequenceFile(zipFile); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); + Fast5Object fast5Object = new Fast5Object(sf); assertEquals(Fast5Object.Fast5Type.SINGLE, fast5Object.getFast5Type()); } @@ -39,8 +50,9 @@ public void testCreateSingleFile() throws IOException { @Test public void testCreateUnknownFile() throws IOException { Path zipFile = createFile(".somethingelse"); - - Fast5Object fast5Object = new Fast5Object(new SequenceFile(zipFile)); + SequenceFile sf = new SequenceFile(zipFile); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); + Fast5Object fast5Object = new Fast5Object(sf); assertEquals(Fast5Object.Fast5Type.UNKNOWN, fast5Object.getFast5Type()); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index d64354071f4..f09219178ff 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -38,7 +38,7 @@ public void setUp() { @Test public void testChecksumCreated() throws IOException { final SequenceFile sf = constructSequenceFile(); - + sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); fileProcessor.process(so); @@ -54,6 +54,7 @@ public void testChecksumCreated() throws IOException { @Test(expected = FileProcessorException.class) public void testFileNotExists() throws IOException { final SequenceFile sf = new SequenceFile(Paths.get("/reallyfakefile")); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 8edbeea806c..a9d87db8d02 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -22,6 +22,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -29,16 +31,19 @@ * * */ + public class GzipFileProcessorTest { private GzipFileProcessor fileProcessor; private SequenceFileRepository sequenceFileRepository; private static final String FILE_CONTENTS = ">test read\nACGTACTCATG"; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); } @Test(expected = FileProcessorException.class) @@ -52,6 +57,7 @@ public void testExceptionBehaviours() throws IOException { Files.copy(uncompressed, out); out.close(); sf.setFile(compressed); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); when(sequenceFileRepository.save(any(SequenceFile.class))).thenThrow(new RuntimeException()); @@ -70,6 +76,7 @@ public void testDeleteOriginalFile() throws IOException { Files.copy(uncompressed, out); out.close(); sf.setFile(compressed); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -85,6 +92,7 @@ public void handleUncompressedFile() throws IOException { // the file processor just shouldn't do *anything*. SequenceFile sf = constructSequenceFile(); Path original = sf.getFile(); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -117,6 +125,7 @@ public void handleCompressedFileWithGzExtension() throws IOException { SingleEndSequenceFile so = new SingleEndSequenceFile(sf); sf.setFile(compressed); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); fileProcessor.process(so); @@ -140,6 +149,8 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { SequenceFile sf = constructSequenceFile(); SequenceFile sfUpdated = new SequenceFile(); sfUpdated.setFile(sf.getFile()); + sfUpdated.setIridaFileStorageUtility(iridaFileStorageUtility); + final Long id = 1L; sf.setId(id); @@ -151,7 +162,8 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { out.close(); sf.setFile(compressed); - + sf.setIridaFileStorageUtility(iridaFileStorageUtility); + SingleEndSequenceFile so = new SingleEndSequenceFile(sf); fileProcessor.process(so); From 26c6867728ab9a3f95978ee45058e3710d0f5984 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Jun 2020 11:02:14 -0500 Subject: [PATCH 062/655] Fixed failing test (gzipped file without an extension) by setting file for sequence file after adding an extension to the file name --- .../bioinformatics/irida/processing/impl/GzipFileProcessor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 596103fb834..0ce5b695d3d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -110,8 +110,10 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); + if (sequenceFile.isGzipped()) { file = addExtensionToFilename(file, GZIP_EXTENSION); + sequenceFile.setFile(file); try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { logger.trace("Handling gzip compressed file."); From 716e4677e843155efa1dfd736ebc263f42c949cf Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Jun 2020 11:44:59 -0500 Subject: [PATCH 063/655] Removed unused imports and added javadoc comments --- .../irida/model/sequenceFile/Fast5Object.java | 2 -- .../irida/model/sequenceFile/SequenceFile.java | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 4475dc916dc..6ff3c22eae9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -12,8 +12,6 @@ import org.slf4j.LoggerFactory; import org.springframework.data.jpa.domain.support.AuditingEntityListener; -import ca.corefacility.bioinformatics.irida.util.FileUtils; - import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; import liquibase.util.file.FilenameUtils; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 429710412d0..7f9e1e2dfc6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -1,7 +1,5 @@ package ca.corefacility.bioinformatics.irida.model.sequenceFile; -import java.io.File; -import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.Date; @@ -306,14 +304,29 @@ public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageU this.iridaFileStorageUtility = iridaFileStorageUtility; } + /** + * Get the file inputstream from the iridaFileStorageUtility + * + * @return file inputstream. + */ public InputStream getFileInputStream() { return iridaFileStorageUtility.getFileInputStream(getFile()); } + /** + * Checks if a file is gzipped in the iridaFileStorageUtility + * + * @return boolean if file is gzipped or not. + */ public boolean isGzipped() throws Exception { return iridaFileStorageUtility.isGzipped(getFile()); } + /** + * Checks if a file exists in the iridaFileStorageUtility + * + * @return boolean if file exists or not. + */ public boolean fileExists() { return iridaFileStorageUtility.fileExists(getFile()); } From 410d81be16b34c7bf54227182a20086e78f2ffc9 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Jun 2020 13:18:13 -0500 Subject: [PATCH 064/655] Removed fileExists from SequenceFile as it's not really a read operation and more of a filesystem operation --- .../config/services/IridaApiServicesConfig.java | 4 ++-- .../irida/model/sequenceFile/SequenceFile.java | 8 -------- .../impl/DefaultFileProcessingChain.java | 11 +++++++---- .../unit/DefaultFileProcessingChainTest.java | 16 ++++++++++------ 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index be1ed9c985a..28fca62b7d7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -293,7 +293,7 @@ public IridaFileStorageUtility iridaFileStorageUtility() { @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, GzipFileProcessor gzipFileProcessor, + QCEntryRepository qcRepository, IridaFileStorageUtility iridaFileStorageUtility, GzipFileProcessor gzipFileProcessor, FastqcFileProcessor fastQcFileProcessor, ChecksumFileProcessor checksumProcessor, CoverageFileProcessor coverageProcessor, AutomatedAnalysisFileProcessor automatedAnalysisFileProcessor) { @@ -307,7 +307,7 @@ public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequenc fileProcessors.remove(gzipFileProcessor); } - return new DefaultFileProcessingChain(sequencingObjectRepository, qcRepository, fileProcessors); + return new DefaultFileProcessingChain(sequencingObjectRepository, qcRepository, iridaFileStorageUtility, fileProcessors); } @Bean(name = "fileProcessingChainExecutor") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 7f9e1e2dfc6..8f33160c44e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -322,12 +322,4 @@ public boolean isGzipped() throws Exception { return iridaFileStorageUtility.isGzipped(getFile()); } - /** - * Checks if a file exists in the iridaFileStorageUtility - * - * @return boolean if file exists or not. - */ - public boolean fileExists() { - return iridaFileStorageUtility.fileExists(getFile()); - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java index 7b1ba9e98fe..938ce14a02e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/DefaultFileProcessingChain.java @@ -12,6 +12,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessingChain; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -35,17 +36,19 @@ public class DefaultFileProcessingChain implements FileProcessingChain { private final SequencingObjectRepository sequencingObjectRepository; private QCEntryRepository qcRepository; + private IridaFileStorageUtility iridaFileStorageUtility; public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, FileProcessor... fileProcessors) { - this(sequencingObjectRepository, qcRepository, Arrays.asList(fileProcessors)); + QCEntryRepository qcRepository, IridaFileStorageUtility iridaFileStorageUtility, FileProcessor... fileProcessors) { + this(sequencingObjectRepository, qcRepository, iridaFileStorageUtility, Arrays.asList(fileProcessors)); } public DefaultFileProcessingChain(SequencingObjectRepository sequencingObjectRepository, - QCEntryRepository qcRepository, List fileProcessors) { + QCEntryRepository qcRepository, IridaFileStorageUtility iridaFileStorageUtility, List fileProcessors) { this.fileProcessors = fileProcessors; this.sequencingObjectRepository = sequencingObjectRepository; this.qcRepository = qcRepository; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -175,7 +178,7 @@ private SequencingObject getSettledSequencingObject(Long sequencingObjectId) thr if(sequencingObject.isPresent()) { Set files = sequencingObject.get().getFiles(); filesNotSettled = files.stream().anyMatch(f -> { - return !f.fileExists(); + return !iridaFileStorageUtility.fileExists(f.getFile()); }); } } while (filesNotSettled); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java index 3e76018557d..ddacfee142f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/DefaultFileProcessingChainTest.java @@ -26,6 +26,8 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.processing.impl.DefaultFileProcessingChain; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sample.QCEntryRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; @@ -38,6 +40,7 @@ public class DefaultFileProcessingChainTest { private SequencingObjectRepository objectRepository; private QCEntryRepository qcRepository; + private IridaFileStorageUtility iridaFileStorageUtility; private SequencingObject seqObject; private Long objectId = 1L; @@ -46,6 +49,7 @@ public class DefaultFileProcessingChainTest { public void setUp() { this.objectRepository = mock(SequencingObjectRepository.class); this.qcRepository = mock(QCEntryRepository.class); + this.iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); seqObject = new NoFileSequencingObject(); when(objectRepository.findById(objectId)).thenReturn(Optional.of(seqObject)); @@ -53,7 +57,7 @@ public void setUp() { @Test(expected = FileProcessorTimeoutException.class) public void testExceedsTimeout() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageUtility); fileProcessingChain.setTimeout(1); fileProcessingChain.setSleepDuration(0); @@ -62,7 +66,7 @@ public void testExceedsTimeout() throws FileProcessorTimeoutException { @Test public void testProcessEmptyChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository); + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageUtility); when(objectRepository.existsById(objectId)).thenReturn(true); fileProcessingChain.launchChain(objectId); @@ -70,7 +74,7 @@ public void testProcessEmptyChain() throws FileProcessorTimeoutException { @Test public void testFailWithContinueChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageUtility, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -84,7 +88,7 @@ public void testFailWithContinueChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFastFailProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageUtility, new FailingFileProcessor()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -96,7 +100,7 @@ public void testFastFailProcessorChain() throws FileProcessorTimeoutException { @Test(expected = FileProcessorException.class) public void testFailOnProcessorChain() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageUtility, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); @@ -106,7 +110,7 @@ public void testFailOnProcessorChain() throws FileProcessorTimeoutException { @Test public void testFailWriteQCEntry() throws FileProcessorTimeoutException { - FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, + FileProcessingChain fileProcessingChain = new DefaultFileProcessingChain(objectRepository, qcRepository, iridaFileStorageUtility, new FailingFileProcessorNoContinue()); when(objectRepository.existsById(objectId)).thenReturn(true); From 83cd0382f10cdd36f6cdd2270e63d2e4c6cb5f17 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 23 Jun 2020 14:04:08 -0500 Subject: [PATCH 065/655] Fixed failing tests --- .../irida/model/sequenceFile/SequenceFile.java | 1 + .../irida/processing/impl/ChecksumFileProcessor.java | 7 +++++-- .../irida/processing/impl/GzipFileProcessor.java | 10 +++++++--- .../impl/unit/ChecksumFileProcessorTest.java | 2 +- .../processing/impl/unit/FastqcFileProcessorTest.java | 8 +++++++- .../processing/impl/unit/GzipFileProcessorTest.java | 4 ++-- .../integration/sequenceFiles/SequenceFilePageIT.java | 2 +- 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 8f33160c44e..79430777374 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -317,6 +317,7 @@ public InputStream getFileInputStream() { * Checks if a file is gzipped in the iridaFileStorageUtility * * @return boolean if file is gzipped or not. + * @throws Exception if the file isn't found */ public boolean isGzipped() throws Exception { return iridaFileStorageUtility.isGzipped(getFile()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index 345c6947ddf..794ea74f2c5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -14,6 +14,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -25,10 +26,12 @@ public class ChecksumFileProcessor implements FileProcessor { private static final Logger logger = LoggerFactory.getLogger(ChecksumFileProcessor.class); private SequenceFileRepository fileRepository; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public ChecksumFileProcessor(SequenceFileRepository fileRepository) { + public ChecksumFileProcessor(SequenceFileRepository fileRepository, IridaFileStorageUtility iridaFileStorageUtility) { this.fileRepository = fileRepository; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -47,7 +50,7 @@ public void process(SequencingObject sequencingObject) { for (SequenceFile file : files) { - try (InputStream is = file.getFileInputStream()) { + try (InputStream is = iridaFileStorageUtility.getFileInputStream(file.getFile())) { String shaDigest = DigestUtils.sha256Hex(is); logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 0ce5b695d3d..3339c3620a6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -17,6 +17,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; /** @@ -37,16 +38,19 @@ public class GzipFileProcessor implements FileProcessor { private final SequenceFileRepository sequenceFileRepository; private boolean disableFileProcessor = false; private boolean removeCompressedFile; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, IridaFileStorageUtility iridaFileStorageUtility) { this.sequenceFileRepository = sequenceFileRepository; removeCompressedFile = false; + this.iridaFileStorageUtility = iridaFileStorageUtility; } - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles, IridaFileStorageUtility iridaFileStorageUtility) { this.sequenceFileRepository = sequenceFileRepository; this.removeCompressedFile = removeCompressedFiles; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -110,7 +114,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); - + //sequenceFile.setIridaFileStorageUtility(iridaFileStorageUtility); if (sequenceFile.isGzipped()) { file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index f09219178ff..1409e1df9c1 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -32,7 +32,7 @@ public class ChecksumFileProcessorTest { public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - fileProcessor = new ChecksumFileProcessor(sequenceFileRepository); + fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index d1ff7b2e4a7..89383ee749b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -66,6 +66,7 @@ public void testHandleFastaFile() throws IOException { Path fasta = Files.createTempFile(null, null); Files.write(fasta, FASTA_FILE_CONTENTS.getBytes()); SequenceFile sf = new SequenceFile(fasta); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); sf.setId(1L); Runtime.getRuntime().addShutdownHook(new DeleteFileOnExit(fasta)); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -76,7 +77,10 @@ public void testHandleFastaFile() throws IOException { @Test public void testHandleFast5File() throws IOException { //ensure we don't process zipped fast5 files - Fast5Object obj = new Fast5Object(new SequenceFile(null)); + SequenceFile sf = new SequenceFile(null); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); + + Fast5Object obj = new Fast5Object(sf); obj.setFast5Type(Fast5Object.Fast5Type.SINGLE); assertTrue("should want to process single fast5 file)", fileProcessor.shouldProcessFile(obj)); @@ -99,6 +103,7 @@ public void testHandleFastqFile() throws IOException, IllegalArgumentException, SequenceFile sf = new SequenceFile(fastq); sf.setId(1L); + sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); try { fileProcessor.process(so); @@ -109,6 +114,7 @@ public void testHandleFastqFile() throws IOException, IllegalArgumentException, verify(sequenceFileRepository).saveMetadata(argument.capture()); SequenceFile updatedFile = argument.getValue(); + updatedFile.setIridaFileStorageUtility(iridaFileStorageUtility); final Field fastqcAnalysis = ReflectionUtils.findField(SequenceFile.class, "fastqcAnalysis"); ReflectionUtils.makeAccessible(fastqcAnalysis); AnalysisFastQC updated = (AnalysisFastQC) fastqcAnalysis.get(updatedFile); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index a9d87db8d02..913a9d821b5 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -42,8 +42,8 @@ public class GzipFileProcessorTest { @Before public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageUtility); } @Test(expected = FileProcessorException.class) @@ -66,7 +66,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageUtility); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index 78cd6f15327..e8410e6de88 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul 18, 2013"; + private static final String FILE_CREATED = "Jul. 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; From 526bd1dc4fa7200ea0f138a8688237c9bb5c0c1e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 23 Jun 2020 14:19:13 -0500 Subject: [PATCH 066/655] Reverted change --- .../irida/processing/impl/GzipFileProcessor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 3339c3620a6..55345459a54 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -114,12 +114,12 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); - //sequenceFile.setIridaFileStorageUtility(iridaFileStorageUtility); - if (sequenceFile.isGzipped()) { + + if (iridaFileStorageUtility.isGzipped(file)) { file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); - try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { + try (GZIPInputStream zippedInputStream = new GZIPInputStream(iridaFileStorageUtility.getFileInputStream(file))) { logger.trace("Handling gzip compressed file."); Path targetDirectory = Files.createTempDirectory(null); From 60f4aac137e7250b86bf891ff709f19eb128d697 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 24 Jun 2020 17:04:28 -0500 Subject: [PATCH 067/655] Removed isgzipped and getfileinputstream from sequencefile as they broke a lot of the tests. --- .../irida/model/sequenceFile/Fast5Object.java | 4 +++- .../model/sequenceFile/SequenceFile.java | 19 ------------------- .../sequenceFiles/SequenceFilePageIT.java | 2 +- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 6ff3c22eae9..48c4ca7f849 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -12,6 +12,8 @@ import org.slf4j.LoggerFactory; import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import ca.corefacility.bioinformatics.irida.util.FileUtils; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; import liquibase.util.file.FilenameUtils; @@ -105,7 +107,7 @@ private Fast5Type setType(SequenceFile sequenceFile) { try { String extension = FilenameUtils.getExtension(file.toString()); - boolean gzipped = sequenceFile.isGzipped(); + boolean gzipped = FileUtils.isGzipped(file); if (gzipped) { type = Fast5Object.Fast5Type.ZIPPED; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 79430777374..5584448e0c9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -304,23 +304,4 @@ public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageU this.iridaFileStorageUtility = iridaFileStorageUtility; } - /** - * Get the file inputstream from the iridaFileStorageUtility - * - * @return file inputstream. - */ - public InputStream getFileInputStream() { - return iridaFileStorageUtility.getFileInputStream(getFile()); - } - - /** - * Checks if a file is gzipped in the iridaFileStorageUtility - * - * @return boolean if file is gzipped or not. - * @throws Exception if the file isn't found - */ - public boolean isGzipped() throws Exception { - return iridaFileStorageUtility.isGzipped(getFile()); - } - } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index e8410e6de88..78cd6f15327 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul. 18, 2013"; + private static final String FILE_CREATED = "Jul 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; From 4fa9df951a8ca908154f048bcc41033f8c9c41de Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 24 Jun 2020 17:16:33 -0500 Subject: [PATCH 068/655] Added getiridafilestorageutility method to sequencefile which is called by the Fast5Object when setting the type --- .../bioinformatics/irida/model/sequenceFile/Fast5Object.java | 3 ++- .../bioinformatics/irida/model/sequenceFile/SequenceFile.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 48c4ca7f849..7c23cdfd88c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -107,7 +108,7 @@ private Fast5Type setType(SequenceFile sequenceFile) { try { String extension = FilenameUtils.getExtension(file.toString()); - boolean gzipped = FileUtils.isGzipped(file); + boolean gzipped = sequenceFile.getIridaFileStorageUtility().isGzipped(file); if (gzipped) { type = Fast5Object.Fast5Type.ZIPPED; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 5584448e0c9..376bfbf016b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -304,4 +304,8 @@ public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageU this.iridaFileStorageUtility = iridaFileStorageUtility; } + public IridaFileStorageUtility getIridaFileStorageUtility() { + return iridaFileStorageUtility; + } + } From 725b73cfebc7da35d2f3b54ba0e9366c1396e854 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 25 Jun 2020 08:07:33 -0500 Subject: [PATCH 069/655] Merged object-store branch and removed unused imports --- .../irida/model/sequenceFile/Fast5Object.java | 3 --- .../irida/model/sequenceFile/SequenceFile.java | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 7c23cdfd88c..92e79caff47 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -12,9 +12,6 @@ import org.slf4j.LoggerFactory; import org.springframework.data.jpa.domain.support.AuditingEntityListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.util.FileUtils; - import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; import liquibase.util.file.FilenameUtils; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 376bfbf016b..d4b5699aaea 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.model.sequenceFile; -import java.io.InputStream; import java.nio.file.Path; import java.util.Date; import java.util.HashMap; @@ -304,6 +303,11 @@ public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageU this.iridaFileStorageUtility = iridaFileStorageUtility; } + /** + * Gets the iridaFileStorageUtility implementation + * + * @return iridaFileStorageUtility implementation + */ public IridaFileStorageUtility getIridaFileStorageUtility() { return iridaFileStorageUtility; } From 3dcf4f3235e7e62530f13147e1d0321600de0065 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 26 Jun 2020 11:26:25 -0500 Subject: [PATCH 070/655] Moved getFileSize out of SequenceFile model and updated Fast5Object model to no longer rely on iridaFileStorageUtility set in SequenceFile. Updated tests. --- .../irida/model/VersionedFileFields.java | 5 -- .../irida/model/assembly/GenomeAssembly.java | 18 ----- .../irida/model/irida/IridaSequenceFile.java | 24 ------- .../irida/model/project/ReferenceFile.java | 10 +-- .../irida/model/sequenceFile/Fast5Object.java | 15 ++-- .../model/sequenceFile/SequenceFile.java | 34 +-------- .../workflow/analysis/AnalysisOutputFile.java | 10 +-- .../listeners/IridaFileStorageListener.java | 36 ---------- .../IridaFileStorageLocalUtilityImpl.java | 7 +- .../filesystem/IridaFileStorageUtility.java | 4 +- .../ria/web/pipelines/PipelineController.java | 32 +++++++-- .../web/samples/SamplesAjaxController.java | 7 +- .../ria/web/samples/SamplesController.java | 50 ++++++++++--- .../irida/ria/web/samples/dto/Fast5Files.java | 29 ++++++++ .../web/samples/dto/GenomeAssemblyFiles.java | 29 ++++++++ .../ria/web/samples/dto/PairedEndFiles.java | 39 ++++++++++ .../ria/web/samples/dto/SingleEndFiles.java | 29 ++++++++ .../bioinformatics/irida/util/FileUtils.java | 9 +++ .../RESTSampleSequenceFilesController.java | 7 +- .../pipelines/types/generic_pipeline.html | 10 +-- .../webapp/pages/samples/sample_files.html | 72 +++++++++---------- .../pages/sequencingRuns/run_files.html | 3 +- .../webapp/pages/templates/_sequenceFile.html | 4 +- .../sequenceFile/unit/Fast5ObjectTest.java | 10 +-- .../impl/unit/ChecksumFileProcessorTest.java | 3 +- .../impl/unit/FastqcFileProcessorTest.java | 7 +- .../impl/unit/GzipFileProcessorTest.java | 6 -- .../analysis/AnalysisDetailsPageIT.java | 4 +- .../samples/SampleDetailsPageIT.java | 2 +- .../sequenceFiles/SequenceFilePageIT.java | 2 +- .../web/pipelines/PipelineControllerTest.java | 6 +- .../samples/SamplesAjaxControllerTest.java | 6 +- .../web/samples/SamplesControllerTest.java | 6 +- .../SampleSequenceFilesControllerTest.java | 7 +- 34 files changed, 304 insertions(+), 238 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java index c492fcad7a0..57329f865d2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java @@ -29,9 +29,4 @@ public interface VersionedFileFields { */ public void incrementFileRevisionNumber(); - /** - * - * @param iridaFileStorageUtility instance of iridaFileStorageUtility to provide to entity - */ - public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 56c9bfb1f65..2945973a397 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -20,8 +20,6 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; import ca.corefacility.bioinformatics.irida.model.joins.impl.SampleGenomeAssemblyJoin; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Lists; @@ -32,13 +30,11 @@ @Entity @Table(name = "genome_assembly") @Inheritance(strategy = InheritanceType.JOINED) -@EntityListeners({ IridaFileStorageListener.class }) @Audited public abstract class GenomeAssembly extends IridaResourceSupport implements IridaThing, IridaSequenceFile, VersionedFileFields { private static final Logger logger = LoggerFactory.getLogger(GenomeAssembly.class); - private static IridaFileStorageUtility iridaFileStorageUtility; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -102,17 +98,6 @@ public void addSampleGenomeAssemblyJoin(SampleGenomeAssemblyJoin join) { } } - /** - * Get human-readable file size. - * - * @return A human-readable file size. - */ - @JsonIgnore - public String getFileSize() { - String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageUtility.getFileSize(getFile()), true); - return size; - } /** * Gets the assembly file. @@ -147,7 +132,4 @@ public boolean equals(Object obj) { return Objects.equals(this.id, other.id) && Objects.equals(this.createdDate, other.createdDate); } - public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { - this.iridaFileStorageUtility = iridaFileStorageUtility; - } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java index 4650e4e03d7..6e30ce58a04 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java @@ -23,28 +23,4 @@ public interface IridaSequenceFile { */ public String getFileName(); - /** - * Get the size of the file. - * - * @return The String representation of the file size - */ - @JsonIgnore - public String getFileSize(); - - /** - * From (http://stackoverflow.com/questions/3758606/how-to-convert-byte-size- into-human-readable-format-in-java) - * - * @param bytes The {@link Long} size of the file in bytes. - * @param si {@link Boolean} true to use si units - * @return A human readable {@link String} representation of the file size. - */ - public static String humanReadableByteCount(long bytes, boolean si) { - int unit = si ? 1000 : 1024; - if (bytes < unit) - return bytes + " B"; - int exp = (int) (Math.log(bytes) / Math.log(unit)); - String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); - return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); - } - } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index 505940521fd..0dcab3c252e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -25,9 +25,7 @@ import ca.corefacility.bioinformatics.irida.model.MutableIridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * A reference file to be associated with a {@link Project}. @@ -37,11 +35,9 @@ @Entity @Table(name = "reference_file") @Audited -@EntityListeners({AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageListener.class}) +@EntityListeners({AuditingEntityListener.class, RelativePathTranslatorListener.class}) public class ReferenceFile implements VersionedFileFields, MutableIridaThing { - private static IridaFileStorageUtility iridaFileStorageUtility; - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -149,8 +145,4 @@ public void setFileLength(Long fileLength) { this.fileLength = fileLength; } - @Override - public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { - this.iridaFileStorageUtility = iridaFileStorageUtility; - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 92e79caff47..ea84f055db7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -42,7 +42,6 @@ protected Fast5Object() { public Fast5Object(SequenceFile file) { this(); this.file = file; - this.fast5Type = setType(file); } /** @@ -94,20 +93,16 @@ public enum Fast5Type { /** * Get the {@link Fast5Type} for this object * - * @param sequenceFile The {@link SequenceFile} to check for type - * @return the detected {@link Fast5Type} + * @param isGzipped if the file type is gzipped or not */ - private Fast5Type setType(SequenceFile sequenceFile) { - Path file = sequenceFile.getFile(); + public void setType(boolean isGzipped) { Fast5Object.Fast5Type type = Fast5Object.Fast5Type.UNKNOWN; try { - String extension = FilenameUtils.getExtension(file.toString()); + String extension = FilenameUtils.getExtension(getFile().getFileName()); - boolean gzipped = sequenceFile.getIridaFileStorageUtility().isGzipped(file); - - if (gzipped) { + if (isGzipped) { type = Fast5Object.Fast5Type.ZIPPED; } else if (extension.equals("fast5")) { type = Fast5Object.Fast5Type.SINGLE; @@ -116,6 +111,6 @@ private Fast5Type setType(SequenceFile sequenceFile) { logger.warn("Problem checking for zipped file. Setting as UNKNOWN type", e); } - return type; + setFast5Type(type); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index d4b5699aaea..916cc56eba2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -26,9 +26,8 @@ import ca.corefacility.bioinformatics.irida.model.remote.RemoteSynchronizable; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; + import com.fasterxml.jackson.annotation.JsonAnyGetter; @@ -42,14 +41,12 @@ @Entity @Table(name = "sequence_file") @Audited -@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class, IridaFileStorageListener.class }) +@EntityListeners({ AuditingEntityListener.class, RelativePathTranslatorListener.class }) public class SequenceFile extends IridaResourceSupport implements MutableIridaThing, Comparable, VersionedFileFields, IridaSequenceFile, RemoteSynchronizable { private static final Logger logger = LoggerFactory.getLogger(SequenceFile.class); - private static IridaFileStorageUtility iridaFileStorageUtility; - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -112,19 +109,6 @@ public String getLabel() { return file.getFileName().toString(); } - /** - * Get the implementation-specific file size. - * - * @return the file size. - */ - @Override - @JsonIgnore - public String getFileSize() { - String size = "N/A"; - size = IridaSequenceFile.humanReadableByteCount(iridaFileStorageUtility.getFileSize(getFile()), true); - return size; - } - /** * Create a new {@link SequenceFile} with the given file Path * @@ -298,18 +282,4 @@ public void setUploadSha256(String uploadSha256) { this.uploadSha256 = uploadSha256; } - @Override - public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { - this.iridaFileStorageUtility = iridaFileStorageUtility; - } - - /** - * Gets the iridaFileStorageUtility implementation - * - * @return iridaFileStorageUtility implementation - */ - public IridaFileStorageUtility getIridaFileStorageUtility() { - return iridaFileStorageUtility; - } - } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 0688915c0ba..f3b360c61f6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -26,10 +26,8 @@ import ca.corefacility.bioinformatics.irida.model.IridaResourceSupport; import ca.corefacility.bioinformatics.irida.model.IridaThing; import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.repositories.entity.listeners.IridaFileStorageListener; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Store file references to files produced by a workflow execution that we @@ -39,11 +37,9 @@ */ @Entity @Table(name = "analysis_output_file") -@EntityListeners({ RelativePathTranslatorListener.class, IridaFileStorageListener.class }) +@EntityListeners({ RelativePathTranslatorListener.class }) public class AnalysisOutputFile extends IridaResourceSupport implements IridaThing, VersionedFileFields { - private static IridaFileStorageUtility iridaFileStorageUtility; - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private final Long id; @@ -193,8 +189,4 @@ public byte[] getBytesForFile() throws IOException { return bytes; } - @Override - public void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { - this.iridaFileStorageUtility = iridaFileStorageUtility; - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java deleted file mode 100644 index 900c510bc8b..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/entity/listeners/IridaFileStorageListener.java +++ /dev/null @@ -1,36 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.entity.listeners; - -import javax.persistence.PostLoad; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.context.support.SpringBeanAutowiringSupport; - -import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; - -/** - * Component implementation to run on a versioned entity after it is has been accessed from the db. - */ -@Component -public class IridaFileStorageListener { - private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageListener.class); - - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - - /** - * After the versioned entity is loaded this method will provide - * the entity access to the iridaFileStorageUtility - * - * @param fileSystemEntity The versioned entity to provide the iridaFileStorageUtility to - */ - @PostLoad - public void afterEntityLoad(final VersionedFileFields fileSystemEntity) { - // Required so that Spring's IoC container can handle the injection - SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); - fileSystemEntity.setIridaFileStorageUtility(iridaFileStorageUtility); - } -} \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 99109f3af2e..f2aa4174b88 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -22,6 +22,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.util.FileUtils; /** * Component implementation of file utitlities for local storage @@ -48,10 +49,10 @@ public File getTemporaryFile(Path file) { * {@inheritDoc} */ @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; + public String getFileSize(Path file) { + String fileSize = "N/A"; try { - fileSize = Files.size(file); + fileSize = FileUtils.humanReadableByteCount(Files.size(file), true); } catch (NoSuchFileException e) { logger.error("Could not find file " + file); } catch (IOException e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 5bd02239b87..0597c7c1e1c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -28,12 +28,12 @@ public interface IridaFileStorageUtility { public File getTemporaryFile(Path file); /** - * Get file size in bytes + * Get file size * * @param file The {@link Path} to the file * @return {@link Long} size of file retrieved from path */ - public Long getFileSize(Path file); + public String getFileSize(Path file); /** * Write file to storage (azure, aws, or local) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java index 817dbf94e13..03a5554ec40 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java @@ -23,11 +23,14 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.IridaWorkflowNamedParameters; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyToolDataService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.BaseController; import ca.corefacility.bioinformatics.irida.ria.web.cart.CartController; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.dto.Pipeline; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.dto.PipelineStartParameters; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.dto.WorkflowParametersToSave; +import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.PairedEndFiles; +import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SingleEndFiles; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; import ca.corefacility.bioinformatics.irida.service.*; import ca.corefacility.bioinformatics.irida.service.user.UserService; @@ -95,6 +98,7 @@ public class PipelineController extends BaseController { private AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor; private GalaxyToolDataService galaxyToolDataService; private EmailController emailController; + private IridaFileStorageUtility iridaFileStorageUtility; /* * CONTROLLERS @@ -108,7 +112,7 @@ public PipelineController(SequencingObjectService sequencingObjectService, CartController cartController, MessageSource messageSource, final WorkflowNamedParametersService namedParameterService, UpdateSamplePermission updateSamplePermission, AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor, - GalaxyToolDataService galaxyToolDataService, EmailController emailController) { + GalaxyToolDataService galaxyToolDataService, EmailController emailController, IridaFileStorageUtility iridaFileStorageUtility) { this.sequencingObjectService = sequencingObjectService; this.referenceFileService = referenceFileService; this.analysisSubmissionService = analysisSubmissionService; @@ -122,6 +126,7 @@ public PipelineController(SequencingObjectService sequencingObjectService, this.analysisSubmissionSampleProcessor = analysisSubmissionSampleProcessor; this.galaxyToolDataService = galaxyToolDataService; this.emailController = emailController; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -225,18 +230,31 @@ public String getSpecifiedPipelinePage(final Model model, Principal principal, L if (description.acceptsPairedSequenceFiles()) { Collection pairs = sequencingObjectService.getSequencesForSampleOfType( sample, SequenceFilePair.class); - files.put("paired_end", pairs.stream() - .map(SampleSequencingObjectJoin::getObject) - .collect(Collectors.toList())); + + List pairedEndFilesList = new ArrayList<>(); + for(SampleSequencingObjectJoin p : pairs) { + SequenceFilePair pair = (SequenceFilePair) p.getObject(); + String forwardFileSize = iridaFileStorageUtility.getFileSize(pair.getForwardSequenceFile().getFile()); + String reverseFileSize = iridaFileStorageUtility.getFileSize(pair.getReverseSequenceFile().getFile()); + pairedEndFilesList.add(new PairedEndFiles(pair, forwardFileSize, reverseFileSize)); + } + + files.put("paired_end", pairedEndFilesList); } + List singleEndFilesList = new ArrayList<>(); // get the single end reads if (description.acceptsSingleSequenceFiles()) { Collection singles = sequencingObjectService.getSequencesForSampleOfType( sample, SingleEndSequenceFile.class); - files.put("single_end", singles.stream() - .map(SampleSequencingObjectJoin::getObject) - .collect(Collectors.toList())); + + for(SampleSequencingObjectJoin p : singles) { + SingleEndSequenceFile singleEndSequenceFile = (SingleEndSequenceFile) p.getObject(); + String fileSize = iridaFileStorageUtility.getFileSize(singleEndSequenceFile.getSequenceFile().getFile()); + singleEndFilesList.add(new SingleEndFiles(singleEndSequenceFile, fileSize)); + } + + files.put("single_end", singleEndFilesList); } sampleMap.put("files", files); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index 05dafa7f8a7..bb8a5f479f1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -22,6 +22,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -36,14 +37,16 @@ public class SamplesAjaxController { private final SequencingObjectService sequencingObjectService; private final GenomeAssemblyService genomeAssemblyService; private final MessageSource messageSource; + private final IridaFileStorageUtility iridaFileStorageUtility; @Autowired public SamplesAjaxController(SampleService sampleService, SequencingObjectService sequencingObjectService, - GenomeAssemblyService genomeAssemblyService, MessageSource messageSource) { + GenomeAssemblyService genomeAssemblyService, MessageSource messageSource, IridaFileStorageUtility iridaFileStorageUtility) { this.sampleService = sampleService; this.sequencingObjectService = sequencingObjectService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -189,6 +192,8 @@ private void createSequenceFileInSample(MultipartFile file, Sample sample) throw */ private void createFast5FileInSample(MultipartFile file, Sample sample) throws IOException { SequenceFile sequenceFile = createSequenceFile(file); + Fast5Object fast5Object = new Fast5Object(sequenceFile); + fast5Object.setType(iridaFileStorageUtility.isGzipped(sequenceFile.getFile())); sequencingObjectService.createSequencingObjectInSample(new Fast5Object(sequenceFile), sample); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 2dff946e829..84f60a04e01 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -35,13 +35,11 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.SampleSequencingObjectJoin; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.user.User; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.BaseController; -import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleDetails; +import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.*; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.ProjectService; @@ -108,6 +106,8 @@ public class SamplesController extends BaseController { private final MetadataTemplateService metadataTemplateService; private final GenomeAssemblyService genomeAssemblyService; + private final IridaFileStorageUtility iridaFileStorageUtility; + private final UpdateSamplePermission updateSamplePermission; private final MessageSource messageSource; @@ -116,7 +116,7 @@ public class SamplesController extends BaseController { public SamplesController(SampleService sampleService, ProjectService projectService, SequencingObjectService sequencingObjectService, UpdateSamplePermission updateSamplePermission, MetadataTemplateService metadataTemplateService, GenomeAssemblyService genomeAssemblyService, - MessageSource messageSource) { + MessageSource messageSource, IridaFileStorageUtility iridaFileStorageUtility) { this.sampleService = sampleService; this.projectService = projectService; this.sequencingObjectService = sequencingObjectService; @@ -124,6 +124,7 @@ public SamplesController(SampleService sampleService, ProjectService projectServ this.metadataTemplateService = metadataTemplateService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /************************************************************************************************ @@ -313,22 +314,49 @@ public String getSampleFiles(final Model model, @PathVariable Long projectId, @P project = projectService.read(projectId); } - // add project to qc entries and filter any unavailable entries + List pairedEndFilesList = new ArrayList<>(); + for (SequencingObject f : filePairs) { + // add project to qc entries and filter any unavailable entries enhanceQcEntries(f, project); + + // create a PairedEndFiles dto which has the pair as well as the forward and reverse file sizes + SequenceFilePair pair = (SequenceFilePair) f; + String forwardFileSize = iridaFileStorageUtility.getFileSize(pair.getForwardSequenceFile().getFile()); + String reverseFileSize = iridaFileStorageUtility.getFileSize(pair.getReverseSequenceFile().getFile()); + pairedEndFilesList.add(new PairedEndFiles(pair, forwardFileSize, reverseFileSize)); } + List fast5FilesList = new ArrayList<>(); + for(SequencingObject f : fast5) { + Fast5Object fast5Object = (Fast5Object) f; + String fileSize = iridaFileStorageUtility.getFileSize(fast5Object.getFile().getFile()); + fast5FilesList.add(new Fast5Files(fast5Object, fileSize)); + } + + List singleEndFilesList = new ArrayList<>(); for (SampleSequencingObjectJoin f : singleFileJoins) { enhanceQcEntries(f.getObject(), project); + + SingleEndSequenceFile singleEndSequenceFile = (SingleEndSequenceFile) f.getObject(); + String fileSize = iridaFileStorageUtility.getFileSize(singleEndSequenceFile.getSequenceFile().getFile()); + singleEndFilesList.add(new SingleEndFiles(singleEndSequenceFile, fileSize)); + } + + List genomeAssemblyFilesList = new ArrayList<>(); + for (GenomeAssembly ga : genomeAssemblies) { + String fileSize = iridaFileStorageUtility.getFileSize(ga.getFile()); + genomeAssemblyFilesList.add(new GenomeAssemblyFiles(ga, fileSize)); } // SequenceFile - model.addAttribute("paired_end", filePairs); - model.addAttribute("single_end", singleFileJoins); - model.addAttribute("fast5", fast5); + model.addAttribute("paired_end", pairedEndFilesList); + model.addAttribute("single_end", singleEndFilesList); + model.addAttribute("fast5", fast5FilesList); + // assemblies - model.addAttribute("assemblies", genomeAssemblies); + model.addAttribute("assemblies", genomeAssemblyFilesList); model.addAttribute(MODEL_ATTR_SAMPLE, sample); model.addAttribute(MODEL_ATTR_CAN_MANAGE_SAMPLE, isSampleModifiable(sample)); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java new file mode 100644 index 00000000000..41c74af4b1e --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java @@ -0,0 +1,29 @@ +package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; + +public class Fast5Files { + private Fast5Object fast5Object; + private String fileSize; + + public Fast5Files(Fast5Object fast5Object, String fileSize) { + this.fast5Object = fast5Object; + this.fileSize = fileSize; + } + + public Fast5Object getFast5Object() { + return fast5Object; + } + + public void setFast5Object(Fast5Object fast5Object) { + this.fast5Object = fast5Object; + } + + public String getFileSize() { + return fileSize; + } + + public void setFileSize(String fileSize) { + this.fileSize = fileSize; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java new file mode 100644 index 00000000000..4a9caecefd4 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java @@ -0,0 +1,29 @@ +package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; + +import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; + +public class GenomeAssemblyFiles { + private GenomeAssembly genomeAssembly; + private String fileSize; + + public GenomeAssemblyFiles(GenomeAssembly genomeAssembly, String fileSize) { + this.genomeAssembly = genomeAssembly; + this.fileSize = fileSize; + } + + public GenomeAssembly getGenomeAssembly() { + return genomeAssembly; + } + + public void setGenomeAssembly(GenomeAssembly genomeAssembly) { + this.genomeAssembly = genomeAssembly; + } + + public String getFileSize() { + return fileSize; + } + + public void setFileSize(String fileSize) { + this.fileSize = fileSize; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java new file mode 100644 index 00000000000..ea147a21e7f --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java @@ -0,0 +1,39 @@ +package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; + +public class PairedEndFiles { + private SequenceFilePair pair; + private String forwardFileSize; + private String reverseFileSize; + + public PairedEndFiles(SequenceFilePair pair, String forwardFileSize, String reverseFileSize) { + this.pair = pair; + this.forwardFileSize = forwardFileSize; + this.reverseFileSize = reverseFileSize; + } + + public SequenceFilePair getPair() { + return pair; + } + + public void setPair(SequenceFilePair pair) { + this.pair = pair; + } + + public String getForwardFileSize() { + return forwardFileSize; + } + + public void setForwardFileSize(String forwardFileSize) { + this.forwardFileSize = forwardFileSize; + } + + public String getReverseFileSize() { + return reverseFileSize; + } + + public void setReverseFileSize(String reverseFileSize) { + this.reverseFileSize = reverseFileSize; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java new file mode 100644 index 00000000000..f203066edf4 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java @@ -0,0 +1,29 @@ +package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; + +public class SingleEndFiles { + private SingleEndSequenceFile singleEndSequenceFile; + private String fileSize; + + public SingleEndFiles(SingleEndSequenceFile singleEndSequenceFile, String fileSize) { + this.singleEndSequenceFile = singleEndSequenceFile; + this.fileSize = fileSize; + } + + public SingleEndSequenceFile getSingleEndSequenceFile() { + return singleEndSequenceFile; + } + + public void setSingleEndSequenceFile(SingleEndSequenceFile singleEndSequenceFile) { + this.singleEndSequenceFile = singleEndSequenceFile; + } + + public String getFileSize() { + return fileSize; + } + + public void setFileSize(String fileSize) { + this.fileSize = fileSize; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index 4f007e932bf..a7a1d2ccc5a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -31,4 +31,13 @@ public static boolean isGzipped(Path file) throws IOException { && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } } + + public static String humanReadableByteCount(long bytes, boolean si) { + int unit = si ? 1000 : 1024; + if (bytes < unit) + return bytes + " B"; + int exp = (int) (Math.log(bytes) / Math.log(unit)); + String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); + return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 25a0ad34807..fdc62d3dafb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -30,6 +30,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -132,17 +133,19 @@ public class RESTSampleSequenceFilesController { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; + private IridaFileStorageUtility iridaFileStorageUtility; protected RESTSampleSequenceFilesController() { } @Autowired public RESTSampleSequenceFilesController(SampleService sampleService, SequencingRunService miseqRunService, - SequencingObjectService sequencingObjectService, AnalysisService analysisService) { + SequencingObjectService sequencingObjectService, AnalysisService analysisService, IridaFileStorageUtility iridaFileStorageUtility) { this.sampleService = sampleService; this.sequencingRunService = miseqRunService; this.sequencingObjectService = sequencingObjectService; this.analysisService = analysisService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -529,6 +532,8 @@ public ModelMap addNewFast5FileToSample(@PathVariable Long sampleId, @RequestPar sf.setFile(target); Fast5Object fast5Object = new Fast5Object(sf); + fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); + if (sequencingRun != null) { if (sequencingRun.getUploadStatus() != SequencingRunUploadStatus.UPLOADING) { throw new IllegalArgumentException("The sequencing run must be in the UPLOADING state to upload data."); diff --git a/src/main/webapp/pages/pipelines/types/generic_pipeline.html b/src/main/webapp/pages/pipelines/types/generic_pipeline.html index 9ca9e284245..4ffff480c84 100644 --- a/src/main/webapp/pages/pipelines/types/generic_pipeline.html +++ b/src/main/webapp/pages/pipelines/types/generic_pipeline.html @@ -265,14 +265,14 @@

- + - + @@ -283,10 +283,10 @@

- + diff --git a/src/main/webapp/pages/samples/sample_files.html b/src/main/webapp/pages/samples/sample_files.html index 1778c2adbb9..0ecce4e2e98 100644 --- a/src/main/webapp/pages/samples/sample_files.html +++ b/src/main/webapp/pages/samples/sample_files.html @@ -51,76 +51,76 @@

ng-controller="FileController as fileCtrl"> + th:with="file=${pair.pair.getForwardSequenceFile()}"> + th:replace="templates/_sequenceFile :: sequenceFileTableLayout (object=${pair.pair},file=${file}, fileSize=${pair.forwardFileSize}, icon='forward',relative='true')">
+ th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${pair.pair.id}, fileId=${file.getId()})}">
+ th:with="file=${pair.pair.getReverseSequenceFile()}"> + th:replace="templates/_sequenceFile :: sequenceFileTableLayout (object=${pair.pair},file=${file},fileSize=${pair.reverseFileSize},icon='reverse',relative='true')">
+ th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${pair.pair.id}, fileId=${file.getId()})}">
+ th:if="${pair.pair.getAutomatedAssembly()}"> _Assembly_ _Status_ + th:text="#{'analysis.state.' + ${pair.pair.getAutomatedAssembly().getAnalysisState()}}"> + th:if="${pair.pair.getSistrTyping()}"> _SISTR_ _Status_ + th:text="#{'analysis.state.' + ${pair.pair.getSistrTyping().getAnalysisState()}}"> - + + th:each="qc : ${pair.pair.getQcEntries()}">
  • - + + th:replace="templates/_sequenceFile :: sequenceFileTableLayout (object=${singleEndFileObject.singleEndSequenceFile},file=${file.getSequenceFile()},fileSize=${singleEndFileObject.fileSize},icon='single',relative='true')">
    + th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${singleEndFileObject.singleEndSequenceFile.getId()}, fileId=${file.getSequenceFile().getId()})}"> @@ -194,13 +194,13 @@

    FAST5 Files

    - +
    + th:text="${file.fileSize}">
    File Size
    @@ -208,7 +208,7 @@

    FAST5 Files

    + th:text="${#dates.format(file.fast5Object.createdDate, dateFormat)}">
    Created Date
    @@ -217,14 +217,14 @@

    FAST5 Files

    + th:attr="download=${file.fast5Object.getLabel()}" + th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${file.fast5Object.getId()}, fileId=${file.fast5Object.getFile().getId()})}"> @@ -252,21 +252,21 @@

    FAST5 Files

    - + + th:href="@{/analysis/{sId}(sId=${assembly.genomeAssembly.analysisSubmission.id})}" + th:text="${assembly.genomeAssembly.label}">
    -
    +
    + th:text="${assembly.fileSize}">
    File Size
    @@ -274,7 +274,7 @@

    FAST5 Files

    + th:text="${#dates.format(assembly.genomeAssembly.getCreatedDate(), dateFormat)}">
    Created Date
    @@ -283,14 +283,14 @@

    FAST5 Files

    + th:attr="download=${assembly.genomeAssembly.label}" + th:href="@{/samples/download/{sampleId}/assembly/{assemblyId}(sampleId=${sample.id}, assemblyId=${assembly.genomeAssembly.id})}">
    diff --git a/src/main/webapp/pages/sequencingRuns/run_files.html b/src/main/webapp/pages/sequencingRuns/run_files.html index bd14b6558d9..54859cf1bbd 100644 --- a/src/main/webapp/pages/sequencingRuns/run_files.html +++ b/src/main/webapp/pages/sequencingRuns/run_files.html @@ -38,8 +38,7 @@ >
    diff --git a/src/main/webapp/pages/templates/_sequenceFile.html b/src/main/webapp/pages/templates/_sequenceFile.html index b2ad6d8b0da..9b9f0e46d86 100644 --- a/src/main/webapp/pages/templates/_sequenceFile.html +++ b/src/main/webapp/pages/templates/_sequenceFile.html @@ -16,7 +16,7 @@
    -
    +
    File Size
    @@ -45,7 +45,7 @@
    -
    +
    File Size
    diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java index b808033bbff..1a6ad98189f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java @@ -31,9 +31,9 @@ public void setUp() { public void testCreateZippedFile() throws IOException { Path zipFile = createZipFile(); SequenceFile sf = new SequenceFile(zipFile); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); - Fast5Object fast5Object = new Fast5Object(sf); + Fast5Object fast5Object = new Fast5Object(sf); + fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); assertEquals(Fast5Object.Fast5Type.ZIPPED, fast5Object.getFast5Type()); } @@ -41,8 +41,9 @@ public void testCreateZippedFile() throws IOException { public void testCreateSingleFile() throws IOException { Path zipFile = createSingleFile(); SequenceFile sf = new SequenceFile(zipFile); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); + Fast5Object fast5Object = new Fast5Object(sf); + fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); assertEquals(Fast5Object.Fast5Type.SINGLE, fast5Object.getFast5Type()); } @@ -51,8 +52,9 @@ public void testCreateSingleFile() throws IOException { public void testCreateUnknownFile() throws IOException { Path zipFile = createFile(".somethingelse"); SequenceFile sf = new SequenceFile(zipFile); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); + Fast5Object fast5Object = new Fast5Object(sf); + fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); assertEquals(Fast5Object.Fast5Type.UNKNOWN, fast5Object.getFast5Type()); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index 1409e1df9c1..66646e49691 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -38,7 +38,7 @@ public void setUp() { @Test public void testChecksumCreated() throws IOException { final SequenceFile sf = constructSequenceFile(); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); + SingleEndSequenceFile so = new SingleEndSequenceFile(sf); fileProcessor.process(so); @@ -54,7 +54,6 @@ public void testChecksumCreated() throws IOException { @Test(expected = FileProcessorException.class) public void testFileNotExists() throws IOException { final SequenceFile sf = new SequenceFile(Paths.get("/reallyfakefile")); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 89383ee749b..2824eb6df1a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -66,7 +66,7 @@ public void testHandleFastaFile() throws IOException { Path fasta = Files.createTempFile(null, null); Files.write(fasta, FASTA_FILE_CONTENTS.getBytes()); SequenceFile sf = new SequenceFile(fasta); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); + sf.setId(1L); Runtime.getRuntime().addShutdownHook(new DeleteFileOnExit(fasta)); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -78,7 +78,6 @@ public void testHandleFastaFile() throws IOException { public void testHandleFast5File() throws IOException { //ensure we don't process zipped fast5 files SequenceFile sf = new SequenceFile(null); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); Fast5Object obj = new Fast5Object(sf); @@ -103,7 +102,7 @@ public void testHandleFastqFile() throws IOException, IllegalArgumentException, SequenceFile sf = new SequenceFile(fastq); sf.setId(1L); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); + SingleEndSequenceFile so = new SingleEndSequenceFile(sf); try { fileProcessor.process(so); @@ -114,7 +113,7 @@ public void testHandleFastqFile() throws IOException, IllegalArgumentException, verify(sequenceFileRepository).saveMetadata(argument.capture()); SequenceFile updatedFile = argument.getValue(); - updatedFile.setIridaFileStorageUtility(iridaFileStorageUtility); + final Field fastqcAnalysis = ReflectionUtils.findField(SequenceFile.class, "fastqcAnalysis"); ReflectionUtils.makeAccessible(fastqcAnalysis); AnalysisFastQC updated = (AnalysisFastQC) fastqcAnalysis.get(updatedFile); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 913a9d821b5..dd92f407b64 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -57,7 +57,6 @@ public void testExceptionBehaviours() throws IOException { Files.copy(uncompressed, out); out.close(); sf.setFile(compressed); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); when(sequenceFileRepository.save(any(SequenceFile.class))).thenThrow(new RuntimeException()); @@ -76,7 +75,6 @@ public void testDeleteOriginalFile() throws IOException { Files.copy(uncompressed, out); out.close(); sf.setFile(compressed); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -92,7 +90,6 @@ public void handleUncompressedFile() throws IOException { // the file processor just shouldn't do *anything*. SequenceFile sf = constructSequenceFile(); Path original = sf.getFile(); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); @@ -125,7 +122,6 @@ public void handleCompressedFileWithGzExtension() throws IOException { SingleEndSequenceFile so = new SingleEndSequenceFile(sf); sf.setFile(compressed); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); fileProcessor.process(so); @@ -149,7 +145,6 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { SequenceFile sf = constructSequenceFile(); SequenceFile sfUpdated = new SequenceFile(); sfUpdated.setFile(sf.getFile()); - sfUpdated.setIridaFileStorageUtility(iridaFileStorageUtility); final Long id = 1L; sf.setId(id); @@ -162,7 +157,6 @@ public void handleCompressedFileWithoutGzExtension() throws IOException { out.close(); sf.setFile(compressed); - sf.setIridaFileStorageUtility(iridaFileStorageUtility); SingleEndSequenceFile so = new SingleEndSequenceFile(sf); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java index fd4766feb5c..99957c7ccb3 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java @@ -66,7 +66,7 @@ public void testAnalysisDetails() { assertEquals("There should be only 6 values for these labels", 6, page.getNumberOfListItemValues()); String[] expectedAnalysisDetails = new String[] { "My Completed Submission", "4", - "SNVPhyl Phylogenomics Pipeline (1.0.1)", "MEDIUM", "Oct 6, 2013, 10:01 AM", "a few seconds" }; + "SNVPhyl Phylogenomics Pipeline (1.0.1)", "MEDIUM", "Oct. 6, 2013, 10:01 a.m.", "a few seconds" }; assertTrue("The correct details are displayed for the analysis", page.analysisDetailsEqual(expectedAnalysisDetails)); } @@ -414,7 +414,7 @@ public void testUnknownPipelineOutput() throws IOException, URISyntaxException, assertEquals("There should be only 6 values for these labels", 6, page.getNumberOfListItemValues()); String[] expectedAnalysisDetails = new String[] { "My Completed Submission UNKNOWN PIPELINE", "14", - "Unknown Pipeline (Unknown Version)", "MEDIUM", "Oct 6, 2013, 10:01 AM", "a few seconds" }; + "Unknown Pipeline (Unknown Version)", "MEDIUM", "Oct. 6, 2013, 10:01 a.m.", "a few seconds" }; assertTrue("The correct details are displayed for the analysis", page.analysisDetailsEqual(expectedAnalysisDetails)); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java index 9b1c5af3c78..b46896841d9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java @@ -18,7 +18,7 @@ @DatabaseSetup("/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplePagesIT.xml") public class SampleDetailsPageIT extends AbstractIridaUIITChromeDriver { private static final Long SAMPLE_ID = 1L; - private static final String SAMPLE_CREATED_DATE = "18 Jul 2013"; + private static final String SAMPLE_CREATED_DATE = "18 Jul. 2013"; private static final String SAMPLE_ORGANISM = "E. coli"; private static final String SAMPLE_LATITUDE = "49.8994"; private static final String SAMPLE_LONGITUDE = "-97.1392"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index 78cd6f15327..e8410e6de88 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul 18, 2013"; + private static final String FILE_CREATED = "Jul. 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java index 0d4747b0038..8f3f96124d9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java @@ -18,6 +18,8 @@ import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyToolDataService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.cart.CartController; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.PipelineController; @@ -70,6 +72,7 @@ public class PipelineControllerTest { private AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor; private GalaxyToolDataService galaxyToolDataService; private TestEmailController emailController; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -84,12 +87,13 @@ public void setUp() { namedParameterService = mock(WorkflowNamedParametersService.class); updateSamplePermission = mock(UpdateSamplePermission.class); analysisSubmissionSampleProcessor = mock(AnalysisSubmissionSampleProcessor.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); galaxyToolDataService = mock(GalaxyToolDataService.class); emailController = mock(TestEmailController.class); controller = new PipelineController(sequencingObjectService, referenceFileService, analysisSubmissionService, workflowsService, projectService, userService, cartController, messageSource, namedParameterService, - updateSamplePermission, analysisSubmissionSampleProcessor, galaxyToolDataService, emailController); + updateSamplePermission, analysisSubmissionSampleProcessor, galaxyToolDataService, emailController, iridaFileStorageUtility); when(messageSource.getMessage(any(), any(), any())).thenReturn(""); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 9b667c25fad..b28dfce390a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -19,6 +19,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; @@ -35,6 +37,7 @@ public class SamplesAjaxControllerTest { private SamplesAjaxController controller; private SequencingObjectService sequencingObjectService; private GenomeAssemblyService genomeAssemblyService; + private IridaFileStorageUtility iridaFileStorageUtility; /* TEST DATA @@ -58,7 +61,8 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource, iridaFileStorageUtility); // Set up mocks when(sampleService.read(SAMPLE.getId())).thenReturn(SAMPLE); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index 349f7149b3a..c6123c87f1e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -29,6 +29,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -61,6 +63,7 @@ public class SamplesControllerTest { private MetadataTemplateService metadataTemplateService; private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -72,8 +75,9 @@ public void setUp() { messageSource = mock(MessageSource.class); updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); controller = new SamplesController(sampleService, projectService, sequencingObjectService, - updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); + updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource, iridaFileStorageUtility); } // ************************************************************************************************ diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index 0211b67c24d..4960766d346 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -40,6 +40,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -61,6 +63,7 @@ public class SampleSequenceFilesControllerTest { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; private SequencingRun sequencingRun; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -69,8 +72,8 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); analysisService = mock(AnalysisService.class); sequencingRun = mock(SequencingRun.class); - - controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService, iridaFileStorageUtility); } @Test From bc0f800320b8908f93591eb059e07ab75b774341 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 26 Jun 2020 13:13:06 -0500 Subject: [PATCH 071/655] Fixed tests. Removed unused imports. Added javadoc comments --- .../bioinformatics/irida/model/VersionedFileFields.java | 1 - .../irida/model/irida/IridaSequenceFile.java | 1 - .../irida/model/sequenceFile/Fast5Object.java | 1 - .../irida/ria/web/samples/dto/Fast5Files.java | 4 ++++ .../irida/ria/web/samples/dto/GenomeAssemblyFiles.java | 4 ++++ .../irida/ria/web/samples/dto/PairedEndFiles.java | 5 +++++ .../irida/ria/web/samples/dto/SingleEndFiles.java | 5 +++++ .../corefacility/bioinformatics/irida/util/FileUtils.java | 7 +++++++ .../ria/integration/analysis/AnalysisDetailsPageIT.java | 4 ++-- .../irida/ria/integration/samples/SampleDetailsPageIT.java | 2 +- .../ria/integration/sequenceFiles/SequenceFilePageIT.java | 2 +- 11 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java index 57329f865d2..c368a56c60e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/VersionedFileFields.java @@ -4,7 +4,6 @@ import javax.persistence.Version; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * An instance of a class may have a property with {@link Version} or may have diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java index 6e30ce58a04..7f1da7f5661 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/irida/IridaSequenceFile.java @@ -2,7 +2,6 @@ import java.nio.file.Path; -import com.fasterxml.jackson.annotation.JsonIgnore; /** * Describes fields that must be made available for a Sequence File in IRIDA diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index ea84f055db7..1f939a7fee6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.model.sequenceFile; -import java.nio.file.Path; import java.util.Date; import java.util.Set; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java index 41c74af4b1e..98dd516b1e2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java @@ -2,6 +2,10 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; +/** + * Used as a response for encapsulating a fast5object its file size + */ + public class Fast5Files { private Fast5Object fast5Object; private String fileSize; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java index 4a9caecefd4..83f9825ae77 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java @@ -2,6 +2,10 @@ import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; +/** + * Used as a response for encapsulating a genome assembly and its file size + */ + public class GenomeAssemblyFiles { private GenomeAssembly genomeAssembly; private String fileSize; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java index ea147a21e7f..b694ee7f67b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java @@ -2,6 +2,11 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +/** + * Used as a response for encapsulating paired end files and their sizes + */ + + public class PairedEndFiles { private SequenceFilePair pair; private String forwardFileSize; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java index f203066edf4..a637ba34467 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java @@ -2,6 +2,11 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +/** + * Used as a response for encapsulating a single end file and its size + */ + + public class SingleEndFiles { private SingleEndSequenceFile singleEndSequenceFile; private String fileSize; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index a7a1d2ccc5a..b4a8b915bee 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -32,6 +32,13 @@ public static boolean isGzipped(Path file) throws IOException { } } + /** + * From (http://stackoverflow.com/questions/3758606/how-to-convert-byte-size- into-human-readable-format-in-java) + * + * @param bytes The {@link Long} size of the file in bytes. + * @param si {@link Boolean} true to use si units + * @return A human readable {@link String} representation of the file size. + */ public static String humanReadableByteCount(long bytes, boolean si) { int unit = si ? 1000 : 1024; if (bytes < unit) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java index 99957c7ccb3..fd4766feb5c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/analysis/AnalysisDetailsPageIT.java @@ -66,7 +66,7 @@ public void testAnalysisDetails() { assertEquals("There should be only 6 values for these labels", 6, page.getNumberOfListItemValues()); String[] expectedAnalysisDetails = new String[] { "My Completed Submission", "4", - "SNVPhyl Phylogenomics Pipeline (1.0.1)", "MEDIUM", "Oct. 6, 2013, 10:01 a.m.", "a few seconds" }; + "SNVPhyl Phylogenomics Pipeline (1.0.1)", "MEDIUM", "Oct 6, 2013, 10:01 AM", "a few seconds" }; assertTrue("The correct details are displayed for the analysis", page.analysisDetailsEqual(expectedAnalysisDetails)); } @@ -414,7 +414,7 @@ public void testUnknownPipelineOutput() throws IOException, URISyntaxException, assertEquals("There should be only 6 values for these labels", 6, page.getNumberOfListItemValues()); String[] expectedAnalysisDetails = new String[] { "My Completed Submission UNKNOWN PIPELINE", "14", - "Unknown Pipeline (Unknown Version)", "MEDIUM", "Oct. 6, 2013, 10:01 a.m.", "a few seconds" }; + "Unknown Pipeline (Unknown Version)", "MEDIUM", "Oct 6, 2013, 10:01 AM", "a few seconds" }; assertTrue("The correct details are displayed for the analysis", page.analysisDetailsEqual(expectedAnalysisDetails)); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java index b46896841d9..9b1c5af3c78 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java @@ -18,7 +18,7 @@ @DatabaseSetup("/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplePagesIT.xml") public class SampleDetailsPageIT extends AbstractIridaUIITChromeDriver { private static final Long SAMPLE_ID = 1L; - private static final String SAMPLE_CREATED_DATE = "18 Jul. 2013"; + private static final String SAMPLE_CREATED_DATE = "18 Jul 2013"; private static final String SAMPLE_ORGANISM = "E. coli"; private static final String SAMPLE_LATITUDE = "49.8994"; private static final String SAMPLE_LONGITUDE = "-97.1392"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index e8410e6de88..78cd6f15327 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul. 18, 2013"; + private static final String FILE_CREATED = "Jul 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; From d57f0adbe638b532bfc4f53b415582077e4a4d55 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 26 Jun 2020 13:28:05 -0500 Subject: [PATCH 072/655] Fixed test --- .../irida/ria/integration/samples/SampleFilesPageIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java index e2880067f00..793aaea0427 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java @@ -65,7 +65,7 @@ public void testDeleteFile() { page.deleteFirstSequenceFile(); assertTrue("Should display a confirmation message that the file was deleted", page.isDeleteConfirmationMessageDisplayed()); - assertEquals("Displays the correct number of sequence files", 3, page.getSequenceFileCount()); + assertEquals("Displays the correct number of sequence files", 2, page.getSequenceFileCount()); } @Test From 94f05f1f4f2024b859a0de913931cf7be63d769d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 26 Jun 2020 15:48:05 -0500 Subject: [PATCH 073/655] Fixed issue with single end file not being deleted. Reverted change to test --- src/main/webapp/pages/samples/sample_files.html | 2 +- .../irida/ria/integration/samples/SampleFilesPageIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/pages/samples/sample_files.html b/src/main/webapp/pages/samples/sample_files.html index 0ecce4e2e98..4ec165eab48 100644 --- a/src/main/webapp/pages/samples/sample_files.html +++ b/src/main/webapp/pages/samples/sample_files.html @@ -143,7 +143,7 @@

    diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java index 793aaea0427..e2880067f00 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleFilesPageIT.java @@ -65,7 +65,7 @@ public void testDeleteFile() { page.deleteFirstSequenceFile(); assertTrue("Should display a confirmation message that the file was deleted", page.isDeleteConfirmationMessageDisplayed()); - assertEquals("Displays the correct number of sequence files", 2, page.getSequenceFileCount()); + assertEquals("Displays the correct number of sequence files", 3, page.getSequenceFileCount()); } @Test From aa4cc7a84226f25c9b0f6681e9b112f60cd62141 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 29 Jun 2020 11:17:26 -0500 Subject: [PATCH 074/655] Updated sequencing runs details page (run_files) to work with getting file sizes from the iridaFileStorageUtility implementation --- .../IridaFileStorageLocalUtilityImpl.java | 4 ++- .../irida/ria/web/samples/dto/Fast5Files.java | 2 +- .../SequencingRunController.java | 26 +++++++++++--- .../dto/SequenceFileDetails.java | 33 ++++++++++++++++++ .../dto/SequencingObjectDetails.java | 34 +++++++++++++++++++ .../pages/sequencingRuns/run_files.html | 15 ++++---- .../unit/web/SequencingRunControllerTest.java | 6 +++- 7 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequenceFileDetails.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequencingObjectDetails.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index f2aa4174b88..db8cb1bb586 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -52,7 +52,9 @@ public File getTemporaryFile(Path file) { public String getFileSize(Path file) { String fileSize = "N/A"; try { - fileSize = FileUtils.humanReadableByteCount(Files.size(file), true); + if(file != null) { + fileSize = FileUtils.humanReadableByteCount(Files.size(file), true); + } } catch (NoSuchFileException e) { logger.error("Could not find file " + file); } catch (IOException e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java index 98dd516b1e2..253e60418f6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java @@ -3,7 +3,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; /** - * Used as a response for encapsulating a fast5object its file size + * Used as a response for encapsulating a fast5object and its file size */ public class Fast5Files { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java index cfb185e78cb..529034d80a9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; @@ -13,7 +12,11 @@ import org.springframework.web.bind.annotation.ResponseBody; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.dto.SequenceFileDetails; +import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.dto.SequencingObjectDetails; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -39,11 +42,13 @@ public class SequencingRunController { private final SequencingRunService sequencingRunService; private final SequencingObjectService objectService; + private final IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public SequencingRunController(SequencingRunService sequencingRunService, SequencingObjectService objectService) { + public SequencingRunController(SequencingRunService sequencingRunService, SequencingObjectService objectService, IridaFileStorageUtility iridaFileStorageUtility) { this.sequencingRunService = sequencingRunService; this.objectService = objectService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -111,9 +116,22 @@ private Model getPageDetails(Long runId, Model model) { Set sequencingObjectsForSequencingRun = objectService .getSequencingObjectsForSequencingRun(run); + /* Gets a list of sequencingObjectDetails which includes the sequencing object id + * and a list of sequence files for that sequencing object with their file sizes + */ + List sequencingObjectList = new ArrayList<>(); + List sequenceFileList = new ArrayList<>(); + for(SequencingObject sequencingObject : sequencingObjectsForSequencingRun) { + Set sequenceFiles = sequencingObject.getFiles(); + for(SequenceFile sequenceFile : sequenceFiles) { + sequenceFileList.add(new SequenceFileDetails(sequenceFile, iridaFileStorageUtility.getFileSize(sequenceFile.getFile()))); + } + sequencingObjectList.add(new SequencingObjectDetails(sequencingObject.getId(), sequenceFileList)); + } + int fileCount = sequencingObjectsForSequencingRun.stream().mapToInt(o -> o.getFiles().size()).sum(); - model.addAttribute("sequencingObjects", sequencingObjectsForSequencingRun); + model.addAttribute("sequencingObjects", sequencingObjectList); model.addAttribute("fileCount", fileCount); model.addAttribute("run", run); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequenceFileDetails.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequenceFileDetails.java new file mode 100644 index 00000000000..254cd87dbd0 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequenceFileDetails.java @@ -0,0 +1,33 @@ +package ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.dto; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + +/** + * Used as a response for encapsulating a sequence file and its file size + */ + +public class SequenceFileDetails { + private SequenceFile sequenceFile; + private String fileSize; + + public SequenceFileDetails(SequenceFile sequenceFile, String fileSize) { + this.sequenceFile = sequenceFile; + this.fileSize = fileSize; + } + + public SequenceFile getSequenceFile() { + return sequenceFile; + } + + public void setSequenceFile(SequenceFile sequenceFile) { + this.sequenceFile = sequenceFile; + } + + public String getFileSize() { + return fileSize; + } + + public void setFileSize(String fileSize) { + this.fileSize = fileSize; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequencingObjectDetails.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequencingObjectDetails.java new file mode 100644 index 00000000000..24d85e551f2 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/dto/SequencingObjectDetails.java @@ -0,0 +1,34 @@ +package ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.dto; + +import java.util.List; + +/** + * Used as a response for encapsulating a sequenceObjectId and a sequence + * file details object (sequence file and its size) + */ + +public class SequencingObjectDetails { + private Long sequencingObjectId; + private List sequenceFileDetailsList; + + public SequencingObjectDetails(Long sequencingObjectId, List sequenceFileDetailsList) { + this.sequencingObjectId = sequencingObjectId; + this.sequenceFileDetailsList = sequenceFileDetailsList; + } + + public Long getSequencingObjectId() { + return sequencingObjectId; + } + + public void setSequencingObjectId(Long sequencingObjectId) { + this.sequencingObjectId = sequencingObjectId; + } + + public List getSequenceFileDetailsList() { + return sequenceFileDetailsList; + } + + public void setSequenceFileDetailsList(List sequenceFileDetailsList) { + this.sequenceFileDetailsList = sequenceFileDetailsList; + } +} diff --git a/src/main/webapp/pages/sequencingRuns/run_files.html b/src/main/webapp/pages/sequencingRuns/run_files.html index 54859cf1bbd..8dfb071843a 100644 --- a/src/main/webapp/pages/sequencingRuns/run_files.html +++ b/src/main/webapp/pages/sequencingRuns/run_files.html @@ -25,26 +25,27 @@ - ID + ID
    diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java index dff3d75c572..b6640dbd947 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java @@ -4,6 +4,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.SequencingRunController; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -23,12 +25,14 @@ public class SequencingRunControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setup() { sequencingRunService = mock(SequencingRunService.class); objectService = mock(SequencingObjectService.class); - controller = new SequencingRunController(sequencingRunService, objectService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + controller = new SequencingRunController(sequencingRunService, objectService, iridaFileStorageUtility); } @Test From 39a1f32d23852993b105c2f79e5d26f4885bd426 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 29 Jun 2020 12:12:43 -0500 Subject: [PATCH 075/655] Fixed typo in template. Fixed logic for creating a new sequenceFileList for each sequencing object --- .../irida/ria/web/sequencingRuns/SequencingRunController.java | 2 +- src/main/webapp/pages/sequencingRuns/run_files.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java index 529034d80a9..ac6bc58d64b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java @@ -120,8 +120,8 @@ private Model getPageDetails(Long runId, Model model) { * and a list of sequence files for that sequencing object with their file sizes */ List sequencingObjectList = new ArrayList<>(); - List sequenceFileList = new ArrayList<>(); for(SequencingObject sequencingObject : sequencingObjectsForSequencingRun) { + List sequenceFileList = new ArrayList<>(); Set sequenceFiles = sequencingObject.getFiles(); for(SequenceFile sequenceFile : sequenceFiles) { sequenceFileList.add(new SequenceFileDetails(sequenceFile, iridaFileStorageUtility.getFileSize(sequenceFile.getFile()))); diff --git a/src/main/webapp/pages/sequencingRuns/run_files.html b/src/main/webapp/pages/sequencingRuns/run_files.html index 8dfb071843a..fd6424053a0 100644 --- a/src/main/webapp/pages/sequencingRuns/run_files.html +++ b/src/main/webapp/pages/sequencingRuns/run_files.html @@ -25,7 +25,7 @@ ID From 660b10c024aff6e9ee61bcf60eb6f05820f8fa01 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 29 Jun 2020 14:24:47 -0500 Subject: [PATCH 076/655] Updated which exceptions are thrown when getting file extensions and appending files. Updated method comments --- .../IridaFileStorageLocalUtilityImpl.java | 14 +++++++------- .../filesystem/IridaFileStorageUtility.java | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index db8cb1bb586..52f28ea2346 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -145,7 +145,7 @@ public boolean isGzipped(Path file) throws IOException { /** * {@inheritDoc} */ - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + public void appendToFile(Path target, SequenceFile file) throws IOException { try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { @@ -154,20 +154,20 @@ public void appendToFile(Path target, SequenceFile file) throws ConcatenateExcep p += in.transferTo(p, l - p, out); } } catch (IOException e) { - throw new ConcatenateException("Could not open input file for reading", e); + throw new IOException("Could not open input file for reading", e); } } catch (IOException e) { - throw new ConcatenateException("Could not open target file for writing", e); + throw new IOException("Could not open target file for writing", e); } } /** * {@inheritDoc} */ - public String getFileExtension(List toConcatenate) throws ConcatenateException { + public String getFileExtension(List sequencingObjects) throws IOException { String selectedExtension = null; - for (SequencingObject object : toConcatenate) { + for (SequencingObject object : sequencingObjects) { for (SequenceFile file : object.getFiles()) { String fileName = file.getFile() @@ -179,7 +179,7 @@ public String getFileExtension(List toConcatenate) t .findFirst(); if (!currentExtensionOpt.isPresent()) { - throw new ConcatenateException("File extension is not valid " + fileName); + throw new IOException("File extension is not valid " + fileName); } String currentExtension = currentExtensionOpt.get(); @@ -187,7 +187,7 @@ public String getFileExtension(List toConcatenate) t if (selectedExtension == null) { selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { - throw new ConcatenateException( + throw new IOException( "Extensions of files to concatenate do not match " + currentExtension + " vs " + selectedExtension); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 0597c7c1e1c..87b9dccb5e1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -95,16 +95,16 @@ public interface IridaFileStorageUtility { * * @param target the {@link Path} to append to * @param file the {@link SequenceFile} to append to the path - * @throws ConcatenateException if there is an error appending the file + * @throws IOException if there is an error appending the file */ - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException; + public void appendToFile(Path target, SequenceFile file) throws IOException; /** - * Get the extension of the files to concatenate + * Get the extension of the files * - * @param toConcatenate The list of {@link SequencingObject} to concatenate + * @param sequencingObjects The list of {@link SequencingObject} to get file extensions for * @return The common extension of the files - * @throws ConcatenateException if the files have different or invalid extensions + * @throws IOException if the files have different or invalid extensions */ - public String getFileExtension(List toConcatenate) throws ConcatenateException; + public String getFileExtension(List sequencingObjects) throws IOException; } From 295deea2ff19f2a7196243e135ac84c9bccde830 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 29 Jun 2020 16:31:21 -0500 Subject: [PATCH 077/655] Updated throws exception --- .../processing/concatenate/SequencingObjectConcatenator.java | 3 ++- .../concatenate/impl/SingleEndSequenceFileConcatenator.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index 2294c83b3b2..e18ef9f3254 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -3,6 +3,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import java.io.IOException; import java.util.List; /** @@ -22,6 +23,6 @@ public abstract class SequencingObjectConcatenator toConcatenate, String filename) - throws ConcatenateException; + throws ConcatenateException, IOException; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index f992b7f2f3b..49ce4ec4632 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -33,7 +33,7 @@ public SingleEndSequenceFileConcatenator(IridaFileStorageUtility iridaFileStorag */ @Override public SingleEndSequenceFile concatenateFiles(List toConcatenate, String filename) - throws ConcatenateException { + throws ConcatenateException, IOException { Path tempFile; String extension = iridaFileStorageUtility.getFileExtension(toConcatenate); From bb473520319daca580837dbc4ed1ca61cb2eab8f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 30 Jun 2020 09:21:56 -0500 Subject: [PATCH 078/655] Added IOException to @throws and added catch block for IOException --- .../concatenate/impl/SequenceFilePairConcatenator.java | 2 +- .../irida/ria/web/samples/SamplesController.java | 5 ++++- .../irida/service/SequencingObjectService.java | 4 +++- .../irida/service/impl/SequencingObjectServiceImpl.java | 4 +++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 3c4b6091f02..7d22d42d460 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -33,7 +33,7 @@ public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtil */ @Override public SequenceFilePair concatenateFiles(List toConcatenate, String filename) - throws ConcatenateException { + throws ConcatenateException, IOException { String extension = iridaFileStorageUtility.getFileExtension(toConcatenate); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 84f60a04e01..ef0ae83167a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -609,7 +609,10 @@ public String concatenateSequenceFiles(@PathVariable Long sampleId, @RequestPara try { sequencingObjectService.concatenateSequences(Lists.newArrayList(readMultiple), filename, sample, removeOriginals); - } catch (ConcatenateException ex) { + } catch (IOException e) { + logger.error("Error reading files: ", e); + } + catch (ConcatenateException ex) { logger.error("Error concatenating files: ", ex); model.addAttribute("concatenateError", true); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java index 8eeae0ce207..7df8dc52d28 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.service; +import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; @@ -140,7 +141,8 @@ public Map getUniqueSamplesForSequencing * the sample * @return the new {@link SampleSequencingObjectJoin} * @throws ConcatenateException if there was an error concatenating the sequences + * @throws IOException if there was an error reading the sequences to concatenate */ public SampleSequencingObjectJoin concatenateSequences(List toJoin, String filename, - Sample targetSample, boolean removeOriginals) throws ConcatenateException; + Sample targetSample, boolean removeOriginals) throws ConcatenateException, IOException; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index 476bb1b713e..ad9031019c4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -27,6 +27,8 @@ import javax.validation.ConstraintViolationException; import javax.validation.Validator; + +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -244,7 +246,7 @@ public Set getSequencingObjectsOfTypeForAn @PreAuthorize("hasPermission(#toJoin, 'canReadSequencingObject') and hasPermission(#targetSample, 'canUpdateSample')") @Transactional public SampleSequencingObjectJoin concatenateSequences(List toJoin, String filename, Sample targetSample, boolean removeOriginals) - throws ConcatenateException { + throws ConcatenateException, IOException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory .getConcatenator(toJoin, iridaFileStorageUtility); From b2d4822cd63d9292b42167d83f42d0f85f2171ac Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 30 Jun 2020 09:44:02 -0500 Subject: [PATCH 079/655] Updated tests to except IOException --- .../impl/SingleEndSequenceFileConcatenatorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index b182ce6364e..bd76c91392e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -86,7 +86,7 @@ public void testConcatenateFilesZipped() throws IOException, ConcatenateExceptio assertEquals("new file should be 2x size of originals", originalLength * 2, newFileSize); } - @Test(expected = ConcatenateException.class) + @Test(expected = IOException.class) public void testConcatenateDifferentFileTypes() throws IOException, ConcatenateException { String newFileName = "newFile"; @@ -99,7 +99,7 @@ public void testConcatenateDifferentFileTypes() throws IOException, ConcatenateE SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); } - @Test(expected = ConcatenateException.class) + @Test(expected = IOException.class) public void testConcatenateBadExtension() throws IOException, ConcatenateException { String newFileName = "newFile"; From ab225e6d847c88ed6b2ecadc1c167d5e6d634b36 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 30 Jun 2020 09:51:44 -0500 Subject: [PATCH 080/655] Updated doc testing --- .../processing/concatenate/SequencingObjectConcatenator.java | 1 + .../filesystem/IridaFileStorageLocalUtilityImpl.java | 1 - .../irida/repositories/filesystem/IridaFileStorageUtility.java | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index e18ef9f3254..0e81d6d4059 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -21,6 +21,7 @@ public abstract class SequencingObjectConcatenator toConcatenate, String filename) throws ConcatenateException, IOException; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 52f28ea2346..28d0599156f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -17,7 +17,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 87b9dccb5e1..1c7133cd241 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -6,7 +6,6 @@ import java.nio.file.Path; import java.util.List; -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; From 165d0c43e10722b416655fbe62e2155040f811fb Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 8 Jul 2020 15:51:49 -0500 Subject: [PATCH 081/655] Moved getFileSize, isGzipped, and getFileInputStream into a static class IridaFiles. Updated tests and templates --- .../irida/model/assembly/GenomeAssembly.java | 9 +++ .../model/assembly/UploadedAssembly.java | 2 +- .../irida/model/project/ReferenceFile.java | 2 +- .../irida/model/sequenceFile/Fast5Object.java | 10 +-- .../model/sequenceFile/SequenceFile.java | 33 ++++++++- .../workflow/analysis/AnalysisOutputFile.java | 2 +- .../impl/ChecksumFileProcessor.java | 6 +- .../processing/impl/GzipFileProcessor.java | 8 +-- .../ria/web/pipelines/PipelineController.java | 30 ++------ .../web/samples/SamplesAjaxController.java | 1 - .../ria/web/samples/SamplesController.java | 35 ++------- .../SequencingRunController.java | 23 +----- .../impl/StaticContextInitializer.java | 28 ++++++++ .../bioinformatics/irida/util/IridaFiles.java | 54 ++++++++++++++ .../RESTSampleSequenceFilesController.java | 1 - .../pipelines/types/generic_pipeline.html | 10 +-- .../webapp/pages/samples/sample_files.html | 72 +++++++++---------- .../pages/sequencingRuns/run_files.html | 16 ++--- .../webapp/pages/templates/_sequenceFile.html | 4 +- .../sequenceFile/unit/Fast5ObjectTest.java | 13 ---- .../impl/unit/ChecksumFileProcessorTest.java | 2 + .../impl/unit/GzipFileProcessorTest.java | 2 + .../unit/web/SequencingRunControllerTest.java | 6 +- .../web/pipelines/PipelineControllerTest.java | 17 +---- .../web/samples/SamplesControllerTest.java | 6 +- 25 files changed, 212 insertions(+), 180 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 2945973a397..9c4fa9fd84e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -20,6 +20,7 @@ import ca.corefacility.bioinformatics.irida.model.VersionedFileFields; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; import ca.corefacility.bioinformatics.irida.model.joins.impl.SampleGenomeAssemblyJoin; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Lists; @@ -98,6 +99,14 @@ public void addSampleGenomeAssemblyJoin(SampleGenomeAssemblyJoin join) { } } + /** + * Gets the assembly file size. + * + * @return The assembly file size. + */ + public String getFileSize() { + return IridaFiles.getFileSize(getFile()); + } /** * Gets the assembly file. diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java index 7420e883954..ccd94f8a8ee 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/UploadedAssembly.java @@ -23,7 +23,7 @@ public class UploadedAssembly extends GenomeAssembly implements VersionedFileFie @NotNull @Column(name = "file_path", unique = true) - protected Path file; + private Path file; @Column(name = "file_revision_number") Long fileRevisionNumber; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index 0dcab3c252e..522246b4165 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -44,7 +44,7 @@ public class ReferenceFile implements VersionedFileFields, MutableIridaThi @Column(name = "filePath", unique = true) @NotNull(message = "{reference.file.file.notnull}") - protected Path file; + private Path file; @CreatedDate @NotNull diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 1f939a7fee6..3d1ea87c8cd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -41,6 +41,7 @@ protected Fast5Object() { public Fast5Object(SequenceFile file) { this(); this.file = file; + this.fast5Type = setType(file); } /** @@ -92,16 +93,17 @@ public enum Fast5Type { /** * Get the {@link Fast5Type} for this object * - * @param isGzipped if the file type is gzipped or not + * @param file if the file + * @return type of fast5object (unknown, zipped, or single) */ - public void setType(boolean isGzipped) { + public Fast5Type setType(SequenceFile file) { Fast5Object.Fast5Type type = Fast5Object.Fast5Type.UNKNOWN; try { String extension = FilenameUtils.getExtension(getFile().getFileName()); - if (isGzipped) { + if (extension.equals("gz")) { type = Fast5Object.Fast5Type.ZIPPED; } else if (extension.equals("fast5")) { type = Fast5Object.Fast5Type.SINGLE; @@ -110,6 +112,6 @@ public void setType(boolean isGzipped) { logger.warn("Problem checking for zipped file. Setting as UNKNOWN type", e); } - setFast5Type(type); + return type; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 916cc56eba2..6dc689d9aaf 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -1,5 +1,7 @@ package ca.corefacility.bioinformatics.irida.model.sequenceFile; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.Date; import java.util.HashMap; @@ -27,8 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; - - +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -282,4 +283,32 @@ public void setUploadSha256(String uploadSha256) { this.uploadSha256 = uploadSha256; } + /** + * Gets the sequence file size. + * + * @return The sequence file size. + */ + public String getFileSize() { + return IridaFiles.getFileSize(file); + } + + /** + * Checks if a file is gzipped. + * + * @return boolean if file is gzipped or not. + * @throws IOException if file cannot be read + */ + public boolean isGzipped() throws IOException { + return IridaFiles.isGzipped(file); + } + + /** + * Gets sequence file input stream + * + * @return returns input stream. + */ + public InputStream getFileInputStream() { + return IridaFiles.getFileInputStream(file); + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index f3b360c61f6..6b38b1cb9c3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -48,7 +48,7 @@ public class AnalysisOutputFile extends IridaResourceSupport implements IridaThi @NotNull(message = "{analysis.output.file.file.notnull}") @com.fasterxml.jackson.annotation.JsonIgnore @org.codehaus.jackson.annotate.JsonIgnore - protected final Path file; + private final Path file; @NotNull @Temporal(TemporalType.TIMESTAMP) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index 794ea74f2c5..629b05c8e95 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -29,6 +29,10 @@ public class ChecksumFileProcessor implements FileProcessor { private IridaFileStorageUtility iridaFileStorageUtility; @Autowired + public ChecksumFileProcessor(SequenceFileRepository fileRepository) { + this.fileRepository = fileRepository; + } + public ChecksumFileProcessor(SequenceFileRepository fileRepository, IridaFileStorageUtility iridaFileStorageUtility) { this.fileRepository = fileRepository; this.iridaFileStorageUtility = iridaFileStorageUtility; @@ -50,7 +54,7 @@ public void process(SequencingObject sequencingObject) { for (SequenceFile file : files) { - try (InputStream is = iridaFileStorageUtility.getFileInputStream(file.getFile())) { + try (InputStream is = file.getFileInputStream()) { String shaDigest = DigestUtils.sha256Hex(is); logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 55345459a54..8981f16e287 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -20,6 +20,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; + /** * Handle gzip-ed files (if necessary). This class partially assumes that gzip * compressed files have the extension ".gz" (not for determining whether or not @@ -41,10 +42,9 @@ public class GzipFileProcessor implements FileProcessor { private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, IridaFileStorageUtility iridaFileStorageUtility) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository) { this.sequenceFileRepository = sequenceFileRepository; removeCompressedFile = false; - this.iridaFileStorageUtility = iridaFileStorageUtility; } public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles, IridaFileStorageUtility iridaFileStorageUtility) { @@ -115,11 +115,11 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); - if (iridaFileStorageUtility.isGzipped(file)) { + if (sequenceFile.isGzipped()) { file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); - try (GZIPInputStream zippedInputStream = new GZIPInputStream(iridaFileStorageUtility.getFileInputStream(file))) { + try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { logger.trace("Handling gzip compressed file."); Path targetDirectory = Files.createTempDirectory(null); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java index 03a5554ec40..2ae32d9c289 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/pipelines/PipelineController.java @@ -23,14 +23,11 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.IridaWorkflowNamedParameters; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyToolDataService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.BaseController; import ca.corefacility.bioinformatics.irida.ria.web.cart.CartController; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.dto.Pipeline; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.dto.PipelineStartParameters; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.dto.WorkflowParametersToSave; -import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.PairedEndFiles; -import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SingleEndFiles; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; import ca.corefacility.bioinformatics.irida.service.*; import ca.corefacility.bioinformatics.irida.service.user.UserService; @@ -98,7 +95,6 @@ public class PipelineController extends BaseController { private AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor; private GalaxyToolDataService galaxyToolDataService; private EmailController emailController; - private IridaFileStorageUtility iridaFileStorageUtility; /* * CONTROLLERS @@ -112,7 +108,7 @@ public PipelineController(SequencingObjectService sequencingObjectService, CartController cartController, MessageSource messageSource, final WorkflowNamedParametersService namedParameterService, UpdateSamplePermission updateSamplePermission, AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor, - GalaxyToolDataService galaxyToolDataService, EmailController emailController, IridaFileStorageUtility iridaFileStorageUtility) { + GalaxyToolDataService galaxyToolDataService, EmailController emailController) { this.sequencingObjectService = sequencingObjectService; this.referenceFileService = referenceFileService; this.analysisSubmissionService = analysisSubmissionService; @@ -126,7 +122,6 @@ public PipelineController(SequencingObjectService sequencingObjectService, this.analysisSubmissionSampleProcessor = analysisSubmissionSampleProcessor; this.galaxyToolDataService = galaxyToolDataService; this.emailController = emailController; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -231,30 +226,19 @@ public String getSpecifiedPipelinePage(final Model model, Principal principal, L Collection pairs = sequencingObjectService.getSequencesForSampleOfType( sample, SequenceFilePair.class); - List pairedEndFilesList = new ArrayList<>(); - for(SampleSequencingObjectJoin p : pairs) { - SequenceFilePair pair = (SequenceFilePair) p.getObject(); - String forwardFileSize = iridaFileStorageUtility.getFileSize(pair.getForwardSequenceFile().getFile()); - String reverseFileSize = iridaFileStorageUtility.getFileSize(pair.getReverseSequenceFile().getFile()); - pairedEndFilesList.add(new PairedEndFiles(pair, forwardFileSize, reverseFileSize)); - } - - files.put("paired_end", pairedEndFilesList); + files.put("paired_end", pairs.stream() + .map(SampleSequencingObjectJoin::getObject) + .collect(Collectors.toList())); } - List singleEndFilesList = new ArrayList<>(); // get the single end reads if (description.acceptsSingleSequenceFiles()) { Collection singles = sequencingObjectService.getSequencesForSampleOfType( sample, SingleEndSequenceFile.class); - for(SampleSequencingObjectJoin p : singles) { - SingleEndSequenceFile singleEndSequenceFile = (SingleEndSequenceFile) p.getObject(); - String fileSize = iridaFileStorageUtility.getFileSize(singleEndSequenceFile.getSequenceFile().getFile()); - singleEndFilesList.add(new SingleEndFiles(singleEndSequenceFile, fileSize)); - } - - files.put("single_end", singleEndFilesList); + files.put("single_end", singles.stream() + .map(SampleSequencingObjectJoin::getObject) + .collect(Collectors.toList())); } sampleMap.put("files", files); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index bb8a5f479f1..ecd0e1ce12f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -193,7 +193,6 @@ private void createSequenceFileInSample(MultipartFile file, Sample sample) throw private void createFast5FileInSample(MultipartFile file, Sample sample) throws IOException { SequenceFile sequenceFile = createSequenceFile(file); Fast5Object fast5Object = new Fast5Object(sequenceFile); - fast5Object.setType(iridaFileStorageUtility.isGzipped(sequenceFile.getFile())); sequencingObjectService.createSequencingObjectInSample(new Fast5Object(sequenceFile), sample); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index ef0ae83167a..ad3c872ff92 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -37,7 +37,6 @@ import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.user.User; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.BaseController; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.*; import ca.corefacility.bioinformatics.irida.security.permissions.sample.UpdateSamplePermission; @@ -106,7 +105,6 @@ public class SamplesController extends BaseController { private final MetadataTemplateService metadataTemplateService; private final GenomeAssemblyService genomeAssemblyService; - private final IridaFileStorageUtility iridaFileStorageUtility; private final UpdateSamplePermission updateSamplePermission; @@ -116,7 +114,7 @@ public class SamplesController extends BaseController { public SamplesController(SampleService sampleService, ProjectService projectService, SequencingObjectService sequencingObjectService, UpdateSamplePermission updateSamplePermission, MetadataTemplateService metadataTemplateService, GenomeAssemblyService genomeAssemblyService, - MessageSource messageSource, IridaFileStorageUtility iridaFileStorageUtility) { + MessageSource messageSource) { this.sampleService = sampleService; this.projectService = projectService; this.sequencingObjectService = sequencingObjectService; @@ -124,7 +122,6 @@ public SamplesController(SampleService sampleService, ProjectService projectServ this.metadataTemplateService = metadataTemplateService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /************************************************************************************************ @@ -319,44 +316,22 @@ public String getSampleFiles(final Model model, @PathVariable Long projectId, @P for (SequencingObject f : filePairs) { // add project to qc entries and filter any unavailable entries enhanceQcEntries(f, project); - - // create a PairedEndFiles dto which has the pair as well as the forward and reverse file sizes - SequenceFilePair pair = (SequenceFilePair) f; - String forwardFileSize = iridaFileStorageUtility.getFileSize(pair.getForwardSequenceFile().getFile()); - String reverseFileSize = iridaFileStorageUtility.getFileSize(pair.getReverseSequenceFile().getFile()); - pairedEndFilesList.add(new PairedEndFiles(pair, forwardFileSize, reverseFileSize)); - } - - List fast5FilesList = new ArrayList<>(); - for(SequencingObject f : fast5) { - Fast5Object fast5Object = (Fast5Object) f; - String fileSize = iridaFileStorageUtility.getFileSize(fast5Object.getFile().getFile()); - fast5FilesList.add(new Fast5Files(fast5Object, fileSize)); } List singleEndFilesList = new ArrayList<>(); for (SampleSequencingObjectJoin f : singleFileJoins) { enhanceQcEntries(f.getObject(), project); - - SingleEndSequenceFile singleEndSequenceFile = (SingleEndSequenceFile) f.getObject(); - String fileSize = iridaFileStorageUtility.getFileSize(singleEndSequenceFile.getSequenceFile().getFile()); - singleEndFilesList.add(new SingleEndFiles(singleEndSequenceFile, fileSize)); } - List genomeAssemblyFilesList = new ArrayList<>(); - for (GenomeAssembly ga : genomeAssemblies) { - String fileSize = iridaFileStorageUtility.getFileSize(ga.getFile()); - genomeAssemblyFilesList.add(new GenomeAssemblyFiles(ga, fileSize)); - } // SequenceFile - model.addAttribute("paired_end", pairedEndFilesList); - model.addAttribute("single_end", singleEndFilesList); - model.addAttribute("fast5", fast5FilesList); + model.addAttribute("paired_end", filePairs); + model.addAttribute("single_end", singleFileJoins); + model.addAttribute("fast5", fast5); // assemblies - model.addAttribute("assemblies", genomeAssemblyFilesList); + model.addAttribute("assemblies", genomeAssemblies); model.addAttribute(MODEL_ATTR_SAMPLE, sample); model.addAttribute(MODEL_ATTR_CAN_MANAGE_SAMPLE, isSampleModifiable(sample)); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java index ac6bc58d64b..2e1a3640b3b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/sequencingRuns/SequencingRunController.java @@ -12,11 +12,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.dto.SequenceFileDetails; -import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.dto.SequencingObjectDetails; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -42,13 +38,11 @@ public class SequencingRunController { private final SequencingRunService sequencingRunService; private final SequencingObjectService objectService; - private final IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public SequencingRunController(SequencingRunService sequencingRunService, SequencingObjectService objectService, IridaFileStorageUtility iridaFileStorageUtility) { + public SequencingRunController(SequencingRunService sequencingRunService, SequencingObjectService objectService) { this.sequencingRunService = sequencingRunService; this.objectService = objectService; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -116,22 +110,9 @@ private Model getPageDetails(Long runId, Model model) { Set sequencingObjectsForSequencingRun = objectService .getSequencingObjectsForSequencingRun(run); - /* Gets a list of sequencingObjectDetails which includes the sequencing object id - * and a list of sequence files for that sequencing object with their file sizes - */ - List sequencingObjectList = new ArrayList<>(); - for(SequencingObject sequencingObject : sequencingObjectsForSequencingRun) { - List sequenceFileList = new ArrayList<>(); - Set sequenceFiles = sequencingObject.getFiles(); - for(SequenceFile sequenceFile : sequenceFiles) { - sequenceFileList.add(new SequenceFileDetails(sequenceFile, iridaFileStorageUtility.getFileSize(sequenceFile.getFile()))); - } - sequencingObjectList.add(new SequencingObjectDetails(sequencingObject.getId(), sequenceFileList)); - } - int fileCount = sequencingObjectsForSequencingRun.stream().mapToInt(o -> o.getFiles().size()).sum(); - model.addAttribute("sequencingObjects", sequencingObjectList); + model.addAttribute("sequencingObjects", sequencingObjectsForSequencingRun); model.addAttribute("fileCount", fileCount); model.addAttribute("run", run); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java new file mode 100644 index 00000000000..b6fc4e26400 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java @@ -0,0 +1,28 @@ +package ca.corefacility.bioinformatics.irida.service.impl; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + +/** + * Allows for setting the iridaFileStorageUtility for static classes + */ + +@Component +public class StaticContextInitializer { + + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + + /** + * Sets the iridaFileStorageUtility in the IridaFiles static class + */ + @PostConstruct + public void init() { + IridaFiles.setIridaFileStorageUtility(this.iridaFileStorageUtility); + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java new file mode 100644 index 00000000000..1d079415f63 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -0,0 +1,54 @@ +package ca.corefacility.bioinformatics.irida.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; + +/** + * Static class which has file object operations that require + * access to the iridaFileStorageUtility but in a static context + */ + +public final class IridaFiles { + + private static IridaFileStorageUtility iridaFileStorageUtility; + + public static void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { + IridaFiles.iridaFileStorageUtility = iridaFileStorageUtility; + } + private IridaFiles() { + } + + /** + * Gets the file size of the file from the iridaFileStorageUtility + * + * @param file The path to the file + * @return file size as a human readable string + */ + public static String getFileSize(Path file) { + return iridaFileStorageUtility.getFileSize(file); + } + + /** + * Checks if the file is gzipped in iridaFileStorageUtility + * + * @param file The path to the file + * @return if file is gzipped or not + * @throws IOException if file cannot be read + */ + public static boolean isGzipped(Path file) throws IOException { + return iridaFileStorageUtility.isGzipped(file); + } + + /** + * Gets the file input stream from iridaFileStorageUtility + * + * @param file The path to the file + * @return the file input stream + */ + public static InputStream getFileInputStream(Path file) { + return iridaFileStorageUtility.getFileInputStream(file); + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index fdc62d3dafb..58436524d3f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -532,7 +532,6 @@ public ModelMap addNewFast5FileToSample(@PathVariable Long sampleId, @RequestPar sf.setFile(target); Fast5Object fast5Object = new Fast5Object(sf); - fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); if (sequencingRun != null) { if (sequencingRun.getUploadStatus() != SequencingRunUploadStatus.UPLOADING) { diff --git a/src/main/webapp/pages/pipelines/types/generic_pipeline.html b/src/main/webapp/pages/pipelines/types/generic_pipeline.html index 4ffff480c84..9ca9e284245 100644 --- a/src/main/webapp/pages/pipelines/types/generic_pipeline.html +++ b/src/main/webapp/pages/pipelines/types/generic_pipeline.html @@ -265,14 +265,14 @@

    - + - + @@ -283,10 +283,10 @@

    - + diff --git a/src/main/webapp/pages/samples/sample_files.html b/src/main/webapp/pages/samples/sample_files.html index 4ec165eab48..9b709fc026b 100644 --- a/src/main/webapp/pages/samples/sample_files.html +++ b/src/main/webapp/pages/samples/sample_files.html @@ -51,76 +51,76 @@

    ng-controller="FileController as fileCtrl"> + th:with="file=${pair.getForwardSequenceFile()}"> + th:replace="templates/_sequenceFile :: sequenceFileTableLayout (object=${pair},file=${file}, icon='forward',relative='true')">
    + th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${pair.id}, fileId=${file.getId()})}">
    + th:with="file=${pair.getReverseSequenceFile()}"> + th:replace="templates/_sequenceFile :: sequenceFileTableLayout (object=${pair},file=${file}, icon='reverse',relative='true')">
    + th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${pair.id}, fileId=${file.getId()})}">
    + th:if="${pair.getAutomatedAssembly()}"> _Assembly_ _Status_ + th:text="#{'analysis.state.' + ${pair.getAutomatedAssembly().getAnalysisState()}}"> + th:if="${pair.getSistrTyping()}"> _SISTR_ _Status_ + th:text="#{'analysis.state.' + ${pair.getSistrTyping().getAnalysisState()}}"> - + + th:each="qc : ${pair.getQcEntries()}">
    @@ -217,14 +217,14 @@

    FAST5 Files

    + th:attr="download=${file.getLabel()}" + th:href="@{/sequenceFiles/download/{objectId}/file/{fileId}(objectId=${file.getId()}, fileId=${file.getFile().getId()})}"> @@ -252,21 +252,21 @@

    FAST5 Files

    - + + th:href="@{/analysis/{sId}(sId=${assembly.analysisSubmission.id})}" + th:text="${assembly.label}">
    -
    +
    + th:text="${assembly.getFileSize()}">
    File Size
    @@ -274,7 +274,7 @@

    FAST5 Files

    + th:text="${#dates.format(assembly.getCreatedDate(), dateFormat)}">
    Created Date
    @@ -283,14 +283,14 @@

    FAST5 Files

    + th:attr="download=${assembly.label}" + th:href="@{/samples/download/{sampleId}/assembly/{assemblyId}(sampleId=${sample.id}, assemblyId=${assembly.id})}">
    diff --git a/src/main/webapp/pages/sequencingRuns/run_files.html b/src/main/webapp/pages/sequencingRuns/run_files.html index fd6424053a0..bd14b6558d9 100644 --- a/src/main/webapp/pages/sequencingRuns/run_files.html +++ b/src/main/webapp/pages/sequencingRuns/run_files.html @@ -25,27 +25,27 @@ - ID + ID
    diff --git a/src/main/webapp/pages/templates/_sequenceFile.html b/src/main/webapp/pages/templates/_sequenceFile.html index 9b9f0e46d86..b2ad6d8b0da 100644 --- a/src/main/webapp/pages/templates/_sequenceFile.html +++ b/src/main/webapp/pages/templates/_sequenceFile.html @@ -16,7 +16,7 @@
    -
    +
    File Size
    @@ -45,7 +45,7 @@
    -
    +
    File Size
    diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java index 1a6ad98189f..4d88a7b9583 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java @@ -5,14 +5,10 @@ import java.nio.file.Path; import java.util.zip.GZIPOutputStream; -import org.junit.Before; import org.junit.Test; import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; - import static org.junit.Assert.assertEquals; /** @@ -20,12 +16,6 @@ */ public class Fast5ObjectTest { private static final String FILE_CONTENTS = "DATA CONTENTS"; - private IridaFileStorageUtility iridaFileStorageUtility; - - @Before - public void setUp() { - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - } @Test public void testCreateZippedFile() throws IOException { @@ -33,7 +23,6 @@ public void testCreateZippedFile() throws IOException { SequenceFile sf = new SequenceFile(zipFile); Fast5Object fast5Object = new Fast5Object(sf); - fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); assertEquals(Fast5Object.Fast5Type.ZIPPED, fast5Object.getFast5Type()); } @@ -43,7 +32,6 @@ public void testCreateSingleFile() throws IOException { SequenceFile sf = new SequenceFile(zipFile); Fast5Object fast5Object = new Fast5Object(sf); - fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); assertEquals(Fast5Object.Fast5Type.SINGLE, fast5Object.getFast5Type()); } @@ -54,7 +42,6 @@ public void testCreateUnknownFile() throws IOException { SequenceFile sf = new SequenceFile(zipFile); Fast5Object fast5Object = new Fast5Object(sf); - fast5Object.setType(iridaFileStorageUtility.isGzipped(sf.getFile())); assertEquals(Fast5Object.Fast5Type.UNKNOWN, fast5Object.getFast5Type()); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java index 66646e49691..0913fab9008 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/ChecksumFileProcessorTest.java @@ -20,6 +20,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; public class ChecksumFileProcessorTest { private ChecksumFileProcessor fileProcessor; @@ -33,6 +34,7 @@ public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); fileProcessor = new ChecksumFileProcessor(sequenceFileRepository, iridaFileStorageUtility); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index dd92f407b64..a3b3616928e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -25,6 +25,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Tests for {@link GzipFileProcessor}. @@ -44,6 +45,7 @@ public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageUtility); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @Test(expected = FileProcessorException.class) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java index b6640dbd947..dff3d75c572 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/SequencingRunControllerTest.java @@ -4,8 +4,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.sequencingRuns.SequencingRunController; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -25,14 +23,12 @@ public class SequencingRunControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; - private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setup() { sequencingRunService = mock(SequencingRunService.class); objectService = mock(SequencingObjectService.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - controller = new SequencingRunController(sequencingRunService, objectService, iridaFileStorageUtility); + controller = new SequencingRunController(sequencingRunService, objectService); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java index 8f3f96124d9..fe3cb9ba428 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/pipelines/PipelineControllerTest.java @@ -18,8 +18,6 @@ import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyToolDataService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.cart.CartController; import ca.corefacility.bioinformatics.irida.ria.web.pipelines.PipelineController; @@ -32,14 +30,7 @@ import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; import ca.corefacility.bioinformatics.irida.service.workflow.WorkflowNamedParametersService; -import org.junit.Before; -import org.junit.Test; -import org.springframework.context.MessageSource; -import org.springframework.ui.ExtendedModelMap; -import java.security.Principal; -import java.util.Locale; -import java.util.UUID; import static org.junit.Assert.*; import static org.mockito.Matchers.any; @@ -47,10 +38,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Created by josh on 15-01-09. @@ -72,7 +59,6 @@ public class PipelineControllerTest { private AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor; private GalaxyToolDataService galaxyToolDataService; private TestEmailController emailController; - private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -87,13 +73,12 @@ public void setUp() { namedParameterService = mock(WorkflowNamedParametersService.class); updateSamplePermission = mock(UpdateSamplePermission.class); analysisSubmissionSampleProcessor = mock(AnalysisSubmissionSampleProcessor.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); galaxyToolDataService = mock(GalaxyToolDataService.class); emailController = mock(TestEmailController.class); controller = new PipelineController(sequencingObjectService, referenceFileService, analysisSubmissionService, workflowsService, projectService, userService, cartController, messageSource, namedParameterService, - updateSamplePermission, analysisSubmissionSampleProcessor, galaxyToolDataService, emailController, iridaFileStorageUtility); + updateSamplePermission, analysisSubmissionSampleProcessor, galaxyToolDataService, emailController); when(messageSource.getMessage(any(), any(), any())).thenReturn(""); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index c6123c87f1e..349f7149b3a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -29,8 +29,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -63,7 +61,6 @@ public class SamplesControllerTest { private MetadataTemplateService metadataTemplateService; private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; - private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -75,9 +72,8 @@ public void setUp() { messageSource = mock(MessageSource.class); updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); controller = new SamplesController(sampleService, projectService, sequencingObjectService, - updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource, iridaFileStorageUtility); + updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); } // ************************************************************************************************ From ccfd5fbcd23cce7a9f7a2abf0c5429b0da2e6b9d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 9 Jul 2020 13:20:19 -0500 Subject: [PATCH 082/655] Added jsonignore annotation so that the methods would not be serialized by jackson --- .../bioinformatics/irida/model/sequenceFile/SequenceFile.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 6dc689d9aaf..eb6f6497434 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -288,6 +288,7 @@ public void setUploadSha256(String uploadSha256) { * * @return The sequence file size. */ + @JsonIgnore public String getFileSize() { return IridaFiles.getFileSize(file); } @@ -298,6 +299,7 @@ public String getFileSize() { * @return boolean if file is gzipped or not. * @throws IOException if file cannot be read */ + @JsonIgnore public boolean isGzipped() throws IOException { return IridaFiles.isGzipped(file); } @@ -307,6 +309,7 @@ public boolean isGzipped() throws IOException { * * @return returns input stream. */ + @JsonIgnore public InputStream getFileInputStream() { return IridaFiles.getFileInputStream(file); } From 69343634fe14ff86c31763c0dca8cd1bf3083350 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 9 Jul 2020 14:25:45 -0500 Subject: [PATCH 083/655] Removed iridaFileStorageUtility from SequencingObjectConcatenator and added appendFile and getFileExtension to IridaFiles static class. Updated tests --- .../SequencingObjectConcatenatorFactory.java | 13 ++++------- .../impl/SequenceFilePairConcatenator.java | 13 ++++------- .../SingleEndSequenceFileConcatenator.java | 10 ++++---- .../impl/SequencingObjectServiceImpl.java | 7 ++---- .../bioinformatics/irida/util/IridaFiles.java | 23 +++++++++++++++++++ ...quencingObjectConcatenatorFactoryTest.java | 23 +++++-------------- .../SequenceFilePairConcatenatorTest.java | 5 ++-- ...SingleEndSequenceFileConcatenatorTest.java | 5 ++-- .../unit/SequencingObjectServiceTest.java | 6 +---- 9 files changed, 52 insertions(+), 53 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java index b6d155081f8..f4cd8472c63 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java @@ -9,7 +9,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Factory class for returning an instance of @@ -22,17 +21,16 @@ public class SequencingObjectConcatenatorFactory { * * @param type the class to get a concatenator for * @param The type this concatenator should act on - * @param iridaFileStorageUtility The file storage component * @return the new {@link SequencingObjectConcatenator} */ @SuppressWarnings("unchecked") - public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageUtility iridaFileStorageUtility) { + public static SequencingObjectConcatenator getConcatenator(Class type) { // return the concatenator for the class if (type.equals(SingleEndSequenceFile.class)) { - return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageUtility); + return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(); } else if (type.equals(SequenceFilePair.class)) { - return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageUtility); + return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(); } else { throw new IllegalArgumentException("No concatenator exists for type " + type); } @@ -43,11 +41,10 @@ public static SequencingObjectConcatenator getCo * {@link SequencingObject}s * * @param objects the {@link SequencingObject}s to get the concatenator for - * @param iridaFileStorageUtility The file storage component * @return the new {@link SequencingObjectConcatenator} */ public static SequencingObjectConcatenator getConcatenator( - Collection objects, IridaFileStorageUtility iridaFileStorageUtility) { + Collection objects) { // get all the classes for the objects Set types = objects.stream().map(Object::getClass).collect(Collectors.toSet()); @@ -61,6 +58,6 @@ public static SequencingObjectConcatenator getConcat @SuppressWarnings("unchecked") Class type = (Class) types.iterator().next(); - return getConcatenator(type, iridaFileStorageUtility); + return getConcatenator(type); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 7d22d42d460..abfc0f424c0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -5,7 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; import java.nio.file.Files; @@ -21,11 +21,8 @@ @Component public class SequenceFilePairConcatenator extends SequencingObjectConcatenator { - private IridaFileStorageUtility iridaFileStorageUtility; - @Autowired - public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtility) { - this.iridaFileStorageUtility = iridaFileStorageUtility; + public SequenceFilePairConcatenator() { } /** @@ -35,7 +32,7 @@ public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtil public SequenceFilePair concatenateFiles(List toConcatenate, String filename) throws ConcatenateException, IOException { - String extension = iridaFileStorageUtility.getFileExtension(toConcatenate); + String extension = IridaFiles.getFileExtension(toConcatenate); // create the filenames with F/R for the forward and reverse files String forwardName = filename + "_R1." + extension; @@ -65,8 +62,8 @@ public SequenceFilePair concatenateFiles(List toConc SequenceFile forwardSequenceFile = pair.getForwardSequenceFile(); SequenceFile reverseSequenceFile = pair.getReverseSequenceFile(); - iridaFileStorageUtility.appendToFile(forwardFile, forwardSequenceFile); - iridaFileStorageUtility.appendToFile(reverseFile, reverseSequenceFile); + IridaFiles.appendToFile(forwardFile, forwardSequenceFile); + IridaFiles.appendToFile(reverseFile, reverseSequenceFile); } // create new SequenceFiles diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 49ce4ec4632..0b1b6b6c4cc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -6,6 +6,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; import java.nio.file.Files; @@ -21,11 +22,8 @@ @Component public class SingleEndSequenceFileConcatenator extends SequencingObjectConcatenator { - private IridaFileStorageUtility iridaFileStorageUtility; - @Autowired - public SingleEndSequenceFileConcatenator(IridaFileStorageUtility iridaFileStorageUtility) { - this.iridaFileStorageUtility = iridaFileStorageUtility; + public SingleEndSequenceFileConcatenator() { } /** @@ -36,7 +34,7 @@ public SingleEndSequenceFile concatenateFiles(List t throws ConcatenateException, IOException { Path tempFile; - String extension = iridaFileStorageUtility.getFileExtension(toConcatenate); + String extension = IridaFiles.getFileExtension(toConcatenate); // create the filename with extension filename = filename + "." + extension; @@ -58,7 +56,7 @@ public SingleEndSequenceFile concatenateFiles(List t SequenceFile forwardSequenceFile = single.getSequenceFile(); - iridaFileStorageUtility.appendToFile(tempFile, forwardSequenceFile); + IridaFiles.appendToFile(tempFile, forwardSequenceFile); } // create the new sequencefile and object diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index ad9031019c4..6bc16a960e1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -12,7 +12,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenatorFactory; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -47,19 +46,17 @@ public class SequencingObjectServiceImpl extends CRUDServiceImpl to throws ConcatenateException, IOException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory - .getConcatenator(toJoin, iridaFileStorageUtility); + .getConcatenator(toJoin); SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 1d079415f63..e5afe3ca690 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -3,7 +3,10 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.List; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** @@ -51,4 +54,24 @@ public static boolean isGzipped(Path file) throws IOException { public static InputStream getFileInputStream(Path file) { return iridaFileStorageUtility.getFileInputStream(file); } + + /** + * Gets the file extension from iridaFileStorageUtility + * + * @param toConcatenate List of sequencingObjects to get file extensions for + * @return the common extension of the files + */ + public static String getFileExtension(List toConcatenate) throws IOException{ + return iridaFileStorageUtility.getFileExtension(toConcatenate); + } + + /** + * Appends a sequence file to another file + * + * @param target The path to the file into which to append + * @param file The sequence file to append + */ + public static void appendToFile(Path target, SequenceFile file) throws IOException { + iridaFileStorageUtility.appendToFile(target, file); + } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index c35ca9382b0..c8cf23d4ce5 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -5,67 +5,56 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.google.common.collect.Sets; -import org.junit.Before; import org.junit.Test; import java.util.Set; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; /** * Unit test for {@link SequencingObjectConcatenatorFactory} */ public class SequencingObjectConcatenatorFactoryTest { - private IridaFileStorageUtility iridaFileStorageUtility; - - @Before - public void setUp() { - iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); - } - @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class, iridaFileStorageUtility); + SingleEndSequenceFile.class); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class, iridaFileStorageUtility); + SequenceFilePair.class); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageUtility); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index b3ced361c23..12f48dd30f7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; import org.junit.Before; @@ -16,7 +17,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; public class SequenceFilePairConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; @@ -28,8 +28,9 @@ public class SequenceFilePairConcatenatorTest { @Before public void setUp() { + concat = new SequenceFilePairConcatenator(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - concat = new SequenceFilePairConcatenator(iridaFileStorageUtility); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index bd76c91392e..442d9f3dc77 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; import org.junit.Before; @@ -16,7 +17,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; public class SingleEndSequenceFileConcatenatorTest { private static final String SEQUENCE = "ACGTACGTN"; @@ -28,8 +28,9 @@ public class SingleEndSequenceFileConcatenatorTest { @Before public void setUp() { + concat = new SingleEndSequenceFileConcatenator(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - concat = new SingleEndSequenceFileConcatenator(iridaFileStorageUtility); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java index 48d9e5118fa..79fbb01eaff 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java @@ -6,8 +6,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -32,7 +30,6 @@ public class SequencingObjectServiceTest { SampleSequencingObjectJoinRepository ssoRepository; SequenceConcatenationRepository concatenationRepository; Validator validator; - IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -40,9 +37,8 @@ public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); ssoRepository = mock(SampleSequencingObjectJoinRepository.class); concatenationRepository = mock(SequenceConcatenationRepository.class); - iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); service = new SequencingObjectServiceImpl(repository, sequenceFileRepository, ssoRepository, - concatenationRepository, validator, iridaFileStorageUtility); + concatenationRepository, validator); } @Test From 8bac91e773dae1a253a0b3570628d4933b8133a4 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 9 Jul 2020 14:51:57 -0500 Subject: [PATCH 084/655] Added missing @throws and removed unused import --- .../concatenate/impl/SingleEndSequenceFileConcatenator.java | 1 - .../ca/corefacility/bioinformatics/irida/util/IridaFiles.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 0b1b6b6c4cc..9bde78e47bc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,7 +5,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index e5afe3ca690..abc226c1ba7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -60,6 +60,7 @@ public static InputStream getFileInputStream(Path file) { * * @param toConcatenate List of sequencingObjects to get file extensions for * @return the common extension of the files + * @throws IOException if file cannot be read */ public static String getFileExtension(List toConcatenate) throws IOException{ return iridaFileStorageUtility.getFileExtension(toConcatenate); @@ -70,6 +71,7 @@ public static String getFileExtension(List toConcate * * @param target The path to the file into which to append * @param file The sequence file to append + * @throws IOException if file cannot be read */ public static void appendToFile(Path target, SequenceFile file) throws IOException { iridaFileStorageUtility.appendToFile(target, file); From 0ee4c8b2ec5bf5570d5b24bf5bb06de7e4a8fb9f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 9 Jul 2020 16:26:42 -0500 Subject: [PATCH 085/655] Removed dtos no longer required. Updated exception handling so only the concatenator classes throw a ConcatenateException --- .../SequencingObjectConcatenator.java | 3 +- .../impl/SequenceFilePairConcatenator.java | 17 +++++-- .../SingleEndSequenceFileConcatenator.java | 15 +++++-- .../web/samples/SamplesAjaxController.java | 5 +-- .../ria/web/samples/SamplesController.java | 3 -- .../irida/ria/web/samples/dto/Fast5Files.java | 33 -------------- .../web/samples/dto/GenomeAssemblyFiles.java | 33 -------------- .../ria/web/samples/dto/PairedEndFiles.java | 44 ------------------- .../ria/web/samples/dto/SingleEndFiles.java | 34 -------------- .../impl/SequencingObjectServiceImpl.java | 2 +- .../bioinformatics/irida/util/IridaFiles.java | 10 ++--- .../RESTSampleSequenceFilesController.java | 5 +-- ...SingleEndSequenceFileConcatenatorTest.java | 4 +- .../samples/SamplesAjaxControllerTest.java | 6 +-- .../SampleSequenceFilesControllerTest.java | 6 +-- 15 files changed, 38 insertions(+), 182 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index 0e81d6d4059..e3e10c45c6e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -21,9 +21,8 @@ public abstract class SequencingObjectConcatenator toConcatenate, String filename) - throws ConcatenateException, IOException; + throws ConcatenateException; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index abfc0f424c0..9235649523b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -30,10 +30,15 @@ public SequenceFilePairConcatenator() { */ @Override public SequenceFilePair concatenateFiles(List toConcatenate, String filename) - throws ConcatenateException, IOException { + throws ConcatenateException { - String extension = IridaFiles.getFileExtension(toConcatenate); + String extension = null; + try { + extension = IridaFiles.getFileExtension(toConcatenate); + } catch (IOException e) { + throw new ConcatenateException("Could not get file extension", e); + } // create the filenames with F/R for the forward and reverse files String forwardName = filename + "_R1." + extension; String reverseName = filename + "_R2." + extension; @@ -62,8 +67,12 @@ public SequenceFilePair concatenateFiles(List toConc SequenceFile forwardSequenceFile = pair.getForwardSequenceFile(); SequenceFile reverseSequenceFile = pair.getReverseSequenceFile(); - IridaFiles.appendToFile(forwardFile, forwardSequenceFile); - IridaFiles.appendToFile(reverseFile, reverseSequenceFile); + try { + IridaFiles.appendToFile(forwardFile, forwardSequenceFile); + IridaFiles.appendToFile(reverseFile, reverseSequenceFile); + } catch (IOException e) { + throw new ConcatenateException("Could not append files", e); + } } // create new SequenceFiles diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 9bde78e47bc..844a0434894 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -30,11 +30,16 @@ public SingleEndSequenceFileConcatenator() { */ @Override public SingleEndSequenceFile concatenateFiles(List toConcatenate, String filename) - throws ConcatenateException, IOException { + throws ConcatenateException { Path tempFile; - String extension = IridaFiles.getFileExtension(toConcatenate); + String extension = null; + try { + extension = IridaFiles.getFileExtension(toConcatenate); + } catch (IOException e) { + throw new ConcatenateException("Could not get file extension", e); + } // create the filename with extension filename = filename + "." + extension; try { @@ -55,7 +60,11 @@ public SingleEndSequenceFile concatenateFiles(List t SequenceFile forwardSequenceFile = single.getSequenceFile(); - IridaFiles.appendToFile(tempFile, forwardSequenceFile); + try { + IridaFiles.appendToFile(tempFile, forwardSequenceFile); + } catch (IOException e) { + throw new ConcatenateException("Could not append file", e); + } } // create the new sequencefile and object diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index ecd0e1ce12f..79ae167df16 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -22,7 +22,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -37,16 +36,14 @@ public class SamplesAjaxController { private final SequencingObjectService sequencingObjectService; private final GenomeAssemblyService genomeAssemblyService; private final MessageSource messageSource; - private final IridaFileStorageUtility iridaFileStorageUtility; @Autowired public SamplesAjaxController(SampleService sampleService, SequencingObjectService sequencingObjectService, - GenomeAssemblyService genomeAssemblyService, MessageSource messageSource, IridaFileStorageUtility iridaFileStorageUtility) { + GenomeAssemblyService genomeAssemblyService, MessageSource messageSource) { this.sampleService = sampleService; this.sequencingObjectService = sequencingObjectService; this.genomeAssemblyService = genomeAssemblyService; this.messageSource = messageSource; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index ad3c872ff92..778fa534ea1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -311,14 +311,11 @@ public String getSampleFiles(final Model model, @PathVariable Long projectId, @P project = projectService.read(projectId); } - List pairedEndFilesList = new ArrayList<>(); - for (SequencingObject f : filePairs) { // add project to qc entries and filter any unavailable entries enhanceQcEntries(f, project); } - List singleEndFilesList = new ArrayList<>(); for (SampleSequencingObjectJoin f : singleFileJoins) { enhanceQcEntries(f.getObject(), project); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java deleted file mode 100644 index 253e60418f6..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/Fast5Files.java +++ /dev/null @@ -1,33 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; - -/** - * Used as a response for encapsulating a fast5object and its file size - */ - -public class Fast5Files { - private Fast5Object fast5Object; - private String fileSize; - - public Fast5Files(Fast5Object fast5Object, String fileSize) { - this.fast5Object = fast5Object; - this.fileSize = fileSize; - } - - public Fast5Object getFast5Object() { - return fast5Object; - } - - public void setFast5Object(Fast5Object fast5Object) { - this.fast5Object = fast5Object; - } - - public String getFileSize() { - return fileSize; - } - - public void setFileSize(String fileSize) { - this.fileSize = fileSize; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java deleted file mode 100644 index 83f9825ae77..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/GenomeAssemblyFiles.java +++ /dev/null @@ -1,33 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; - -import ca.corefacility.bioinformatics.irida.model.assembly.GenomeAssembly; - -/** - * Used as a response for encapsulating a genome assembly and its file size - */ - -public class GenomeAssemblyFiles { - private GenomeAssembly genomeAssembly; - private String fileSize; - - public GenomeAssemblyFiles(GenomeAssembly genomeAssembly, String fileSize) { - this.genomeAssembly = genomeAssembly; - this.fileSize = fileSize; - } - - public GenomeAssembly getGenomeAssembly() { - return genomeAssembly; - } - - public void setGenomeAssembly(GenomeAssembly genomeAssembly) { - this.genomeAssembly = genomeAssembly; - } - - public String getFileSize() { - return fileSize; - } - - public void setFileSize(String fileSize) { - this.fileSize = fileSize; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java deleted file mode 100644 index b694ee7f67b..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/PairedEndFiles.java +++ /dev/null @@ -1,44 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; - -/** - * Used as a response for encapsulating paired end files and their sizes - */ - - -public class PairedEndFiles { - private SequenceFilePair pair; - private String forwardFileSize; - private String reverseFileSize; - - public PairedEndFiles(SequenceFilePair pair, String forwardFileSize, String reverseFileSize) { - this.pair = pair; - this.forwardFileSize = forwardFileSize; - this.reverseFileSize = reverseFileSize; - } - - public SequenceFilePair getPair() { - return pair; - } - - public void setPair(SequenceFilePair pair) { - this.pair = pair; - } - - public String getForwardFileSize() { - return forwardFileSize; - } - - public void setForwardFileSize(String forwardFileSize) { - this.forwardFileSize = forwardFileSize; - } - - public String getReverseFileSize() { - return reverseFileSize; - } - - public void setReverseFileSize(String reverseFileSize) { - this.reverseFileSize = reverseFileSize; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java deleted file mode 100644 index a637ba34467..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SingleEndFiles.java +++ /dev/null @@ -1,34 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; - -/** - * Used as a response for encapsulating a single end file and its size - */ - - -public class SingleEndFiles { - private SingleEndSequenceFile singleEndSequenceFile; - private String fileSize; - - public SingleEndFiles(SingleEndSequenceFile singleEndSequenceFile, String fileSize) { - this.singleEndSequenceFile = singleEndSequenceFile; - this.fileSize = fileSize; - } - - public SingleEndSequenceFile getSingleEndSequenceFile() { - return singleEndSequenceFile; - } - - public void setSingleEndSequenceFile(SingleEndSequenceFile singleEndSequenceFile) { - this.singleEndSequenceFile = singleEndSequenceFile; - } - - public String getFileSize() { - return fileSize; - } - - public void setFileSize(String fileSize) { - this.fileSize = fileSize; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index 6bc16a960e1..d79ef0cf0b7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -243,7 +243,7 @@ public Set getSequencingObjectsOfTypeForAn @PreAuthorize("hasPermission(#toJoin, 'canReadSequencingObject') and hasPermission(#targetSample, 'canUpdateSample')") @Transactional public SampleSequencingObjectJoin concatenateSequences(List toJoin, String filename, Sample targetSample, boolean removeOriginals) - throws ConcatenateException, IOException { + throws ConcatenateException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory .getConcatenator(toJoin); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index abc226c1ba7..e97aa488bef 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -58,12 +58,12 @@ public static InputStream getFileInputStream(Path file) { /** * Gets the file extension from iridaFileStorageUtility * - * @param toConcatenate List of sequencingObjects to get file extensions for + * @param files List of sequencingObjects to get file extensions for * @return the common extension of the files - * @throws IOException if file cannot be read + * @throws IOException if file(s) cannot be read */ - public static String getFileExtension(List toConcatenate) throws IOException{ - return iridaFileStorageUtility.getFileExtension(toConcatenate); + public static String getFileExtension(List files) throws IOException { + return iridaFileStorageUtility.getFileExtension(files); } /** @@ -71,7 +71,7 @@ public static String getFileExtension(List toConcate * * @param target The path to the file into which to append * @param file The sequence file to append - * @throws IOException if file cannot be read + * @throws IOException if file cannot be read/written */ public static void appendToFile(Path target, SequenceFile file) throws IOException { iridaFileStorageUtility.appendToFile(target, file); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 58436524d3f..bda188e9da5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -30,7 +30,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.*; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -133,19 +132,17 @@ public class RESTSampleSequenceFilesController { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; - private IridaFileStorageUtility iridaFileStorageUtility; protected RESTSampleSequenceFilesController() { } @Autowired public RESTSampleSequenceFilesController(SampleService sampleService, SequencingRunService miseqRunService, - SequencingObjectService sequencingObjectService, AnalysisService analysisService, IridaFileStorageUtility iridaFileStorageUtility) { + SequencingObjectService sequencingObjectService, AnalysisService analysisService) { this.sampleService = sampleService; this.sequencingRunService = miseqRunService; this.sequencingObjectService = sequencingObjectService; this.analysisService = analysisService; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 442d9f3dc77..758908cb7f5 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -87,7 +87,7 @@ public void testConcatenateFilesZipped() throws IOException, ConcatenateExceptio assertEquals("new file should be 2x size of originals", originalLength * 2, newFileSize); } - @Test(expected = IOException.class) + @Test(expected = ConcatenateException.class) public void testConcatenateDifferentFileTypes() throws IOException, ConcatenateException { String newFileName = "newFile"; @@ -100,7 +100,7 @@ public void testConcatenateDifferentFileTypes() throws IOException, ConcatenateE SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); } - @Test(expected = IOException.class) + @Test(expected = ConcatenateException.class) public void testConcatenateBadExtension() throws IOException, ConcatenateException { String newFileName = "newFile"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index b28dfce390a..9b667c25fad 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -19,8 +19,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; @@ -37,7 +35,6 @@ public class SamplesAjaxControllerTest { private SamplesAjaxController controller; private SequencingObjectService sequencingObjectService; private GenomeAssemblyService genomeAssemblyService; - private IridaFileStorageUtility iridaFileStorageUtility; /* TEST DATA @@ -61,8 +58,7 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource, iridaFileStorageUtility); + controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); // Set up mocks when(sampleService.read(SAMPLE.getId())).thenReturn(SAMPLE); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index 4960766d346..ea820d94e17 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -40,8 +40,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisFastQC; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; @@ -63,7 +61,6 @@ public class SampleSequenceFilesControllerTest { private SequencingObjectService sequencingObjectService; private AnalysisService analysisService; private SequencingRun sequencingRun; - private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -72,8 +69,7 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); analysisService = mock(AnalysisService.class); sequencingRun = mock(SequencingRun.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService, iridaFileStorageUtility); + controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService,analysisService); } @Test From 1675e3a782e0f16ec957be2e7132f345dcc47039 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 9 Jul 2020 16:43:23 -0500 Subject: [PATCH 086/655] Removed unused imports --- .../processing/concatenate/SequencingObjectConcatenator.java | 1 - .../irida/service/impl/SequencingObjectServiceImpl.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index e3e10c45c6e..2294c83b3b2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -3,7 +3,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import java.io.IOException; import java.util.List; /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index d79ef0cf0b7..e6c369c3158 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -27,7 +27,6 @@ import javax.validation.ConstraintViolationException; import javax.validation.Validator; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; From 97e0014e1c5aa5584ad4b4f07bf67f3d54fbaf70 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 10 Jul 2020 12:01:50 -0500 Subject: [PATCH 087/655] Removed staticcontextinitalizer and instead set the IridaFiles iridaFileStorageUtility when creating the bean. Removed appendFile from IridaFiles as its an actual file operation and instead the iridaFileStorageUtility append file method is called. Updated tests --- .../services/IridaApiServicesConfig.java | 5 +++- .../irida/model/sequenceFile/Fast5Object.java | 4 +-- .../SequencingObjectConcatenatorFactory.java | 13 +++++---- .../impl/SequenceFilePairConcatenator.java | 14 +++++----- .../SingleEndSequenceFileConcatenator.java | 12 ++++---- .../impl/SequencingObjectServiceImpl.java | 7 +++-- .../impl/StaticContextInitializer.java | 28 ------------------- .../bioinformatics/irida/util/IridaFiles.java | 11 -------- ...quencingObjectConcatenatorFactoryTest.java | 24 ++++++++++++---- .../SequenceFilePairConcatenatorTest.java | 2 +- ...SingleEndSequenceFileConcatenatorTest.java | 2 +- .../unit/SequencingObjectServiceTest.java | 6 +++- 12 files changed, 57 insertions(+), 71 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 28fca62b7d7..bceb431d6a4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -24,6 +24,7 @@ import ca.corefacility.bioinformatics.irida.service.impl.InMemoryTaxonomyService; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.AnalysisSubmissionCleanupServiceImpl; import ca.corefacility.bioinformatics.irida.service.user.UserService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.util.IridaPluginMessageSource; import com.google.common.collect.ImmutableList; @@ -288,7 +289,9 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageUtility") public IridaFileStorageUtility iridaFileStorageUtility() { - return new IridaFileStorageLocalUtilityImpl(); + IridaFileStorageUtility iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + return iridaFileStorageUtility; } @Bean(name = "uploadFileProcessingChain") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 3d1ea87c8cd..a5f81f28e04 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -96,14 +96,14 @@ public enum Fast5Type { * @param file if the file * @return type of fast5object (unknown, zipped, or single) */ - public Fast5Type setType(SequenceFile file) { + private Fast5Type setType(SequenceFile file) { Fast5Object.Fast5Type type = Fast5Object.Fast5Type.UNKNOWN; try { String extension = FilenameUtils.getExtension(getFile().getFileName()); - if (extension.equals("gz")) { + if (file.isGzipped()) { type = Fast5Object.Fast5Type.ZIPPED; } else if (extension.equals("fast5")) { type = Fast5Object.Fast5Type.SINGLE; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java index f4cd8472c63..b6d155081f8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactory.java @@ -9,6 +9,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; /** * Factory class for returning an instance of @@ -21,16 +22,17 @@ public class SequencingObjectConcatenatorFactory { * * @param type the class to get a concatenator for * @param The type this concatenator should act on + * @param iridaFileStorageUtility The file storage component * @return the new {@link SequencingObjectConcatenator} */ @SuppressWarnings("unchecked") - public static SequencingObjectConcatenator getConcatenator(Class type) { + public static SequencingObjectConcatenator getConcatenator(Class type, IridaFileStorageUtility iridaFileStorageUtility) { // return the concatenator for the class if (type.equals(SingleEndSequenceFile.class)) { - return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(); + return (SequencingObjectConcatenator) new SingleEndSequenceFileConcatenator(iridaFileStorageUtility); } else if (type.equals(SequenceFilePair.class)) { - return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(); + return (SequencingObjectConcatenator) new SequenceFilePairConcatenator(iridaFileStorageUtility); } else { throw new IllegalArgumentException("No concatenator exists for type " + type); } @@ -41,10 +43,11 @@ public static SequencingObjectConcatenator getCo * {@link SequencingObject}s * * @param objects the {@link SequencingObject}s to get the concatenator for + * @param iridaFileStorageUtility The file storage component * @return the new {@link SequencingObjectConcatenator} */ public static SequencingObjectConcatenator getConcatenator( - Collection objects) { + Collection objects, IridaFileStorageUtility iridaFileStorageUtility) { // get all the classes for the objects Set types = objects.stream().map(Object::getClass).collect(Collectors.toSet()); @@ -58,6 +61,6 @@ public static SequencingObjectConcatenator getConcat @SuppressWarnings("unchecked") Class type = (Class) types.iterator().next(); - return getConcatenator(type); + return getConcatenator(type, iridaFileStorageUtility); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index 9235649523b..ade211d4db0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; @@ -12,17 +13,16 @@ import java.nio.file.Path; import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; /** * {@link SequencingObjectConcatenator} for {@link SequenceFilePair}s */ -@Component public class SequenceFilePairConcatenator extends SequencingObjectConcatenator { - @Autowired - public SequenceFilePairConcatenator() { + private IridaFileStorageUtility iridaFileStorageUtility; + + public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -68,8 +68,8 @@ public SequenceFilePair concatenateFiles(List toConc SequenceFile reverseSequenceFile = pair.getReverseSequenceFile(); try { - IridaFiles.appendToFile(forwardFile, forwardSequenceFile); - IridaFiles.appendToFile(reverseFile, reverseSequenceFile); + iridaFileStorageUtility.appendToFile(forwardFile, forwardSequenceFile); + iridaFileStorageUtility.appendToFile(reverseFile, reverseSequenceFile); } catch (IOException e) { throw new ConcatenateException("Could not append files", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 844a0434894..67b1d3cc205 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; @@ -12,17 +13,16 @@ import java.nio.file.Path; import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; /** * {@link SequenceFilePairConcatenator} for {@link SingleEndSequenceFile}s */ -@Component public class SingleEndSequenceFileConcatenator extends SequencingObjectConcatenator { - @Autowired - public SingleEndSequenceFileConcatenator() { + private IridaFileStorageUtility iridaFileStorageUtility; + + public SingleEndSequenceFileConcatenator(IridaFileStorageUtility iridaFileStorageUtility) { + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -61,7 +61,7 @@ public SingleEndSequenceFile concatenateFiles(List t SequenceFile forwardSequenceFile = single.getSequenceFile(); try { - IridaFiles.appendToFile(tempFile, forwardSequenceFile); + iridaFileStorageUtility.appendToFile(tempFile, forwardSequenceFile); } catch (IOException e) { throw new ConcatenateException("Could not append file", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index e6c369c3158..27d62293fb8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -12,6 +12,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenatorFactory; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -45,17 +46,19 @@ public class SequencingObjectServiceImpl extends CRUDServiceImpl to throws ConcatenateException { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory - .getConcatenator(toJoin); + .getConcatenator(toJoin, iridaFileStorageUtility); SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java deleted file mode 100644 index b6fc4e26400..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/StaticContextInitializer.java +++ /dev/null @@ -1,28 +0,0 @@ -package ca.corefacility.bioinformatics.irida.service.impl; - -import javax.annotation.PostConstruct; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; - -/** - * Allows for setting the iridaFileStorageUtility for static classes - */ - -@Component -public class StaticContextInitializer { - - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - - /** - * Sets the iridaFileStorageUtility in the IridaFiles static class - */ - @PostConstruct - public void init() { - IridaFiles.setIridaFileStorageUtility(this.iridaFileStorageUtility); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index e97aa488bef..21efbc244ac 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -5,7 +5,6 @@ import java.nio.file.Path; import java.util.List; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; @@ -66,14 +65,4 @@ public static String getFileExtension(List files) th return iridaFileStorageUtility.getFileExtension(files); } - /** - * Appends a sequence file to another file - * - * @param target The path to the file into which to append - * @param file The sequence file to append - * @throws IOException if file cannot be read/written - */ - public static void appendToFile(Path target, SequenceFile file) throws IOException { - iridaFileStorageUtility.appendToFile(target, file); - } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java index c8cf23d4ce5..3e4af3f12b9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenatorFactoryTest.java @@ -5,56 +5,68 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SequenceFilePairConcatenator; import ca.corefacility.bioinformatics.irida.processing.concatenate.impl.SingleEndSequenceFileConcatenator; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.google.common.collect.Sets; +import org.junit.Before; import org.junit.Test; import java.util.Set; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; /** * Unit test for {@link SequencingObjectConcatenatorFactory} */ public class SequencingObjectConcatenatorFactoryTest { + private IridaFileStorageUtility iridaFileStorageUtility; + + @Before + public void setUp() { + iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); + } + + @Test public void testGetConcatenatorSingle() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SingleEndSequenceFile.class); + SingleEndSequenceFile.class, iridaFileStorageUtility); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPair() { SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator( - SequenceFilePair.class); + SequenceFilePair.class, iridaFileStorageUtility); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorError() { - SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class); + SequencingObjectConcatenatorFactory.getConcatenator(SequencingObject.class, iridaFileStorageUtility); } @Test public void testGetConcatenatorSingleCollection() { Set fileSet = Sets.newHashSet(new SingleEndSequenceFile(null)); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); assertTrue(concatenator instanceof SingleEndSequenceFileConcatenator); } @Test public void testGetConcatenatorPairCollection() { Set fileSet = Sets.newHashSet(new SequenceFilePair()); - SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); assertTrue(concatenator instanceof SequenceFilePairConcatenator); } @Test(expected = IllegalArgumentException.class) public void testGetConcatenatorMixedError() { Set fileSet = Sets.newHashSet(new SequenceFilePair(), new SingleEndSequenceFile(null)); - SequencingObjectConcatenatorFactory.getConcatenator(fileSet); + SequencingObjectConcatenatorFactory.getConcatenator(fileSet, iridaFileStorageUtility); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 12f48dd30f7..ba26c80d185 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -28,9 +28,9 @@ public class SequenceFilePairConcatenatorTest { @Before public void setUp() { - concat = new SequenceFilePairConcatenator(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + concat = new SequenceFilePairConcatenator(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 758908cb7f5..5652308314c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -28,9 +28,9 @@ public class SingleEndSequenceFileConcatenatorTest { @Before public void setUp() { - concat = new SingleEndSequenceFileConcatenator(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + concat = new SingleEndSequenceFileConcatenator(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java index 79fbb01eaff..48d9e5118fa 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/SequencingObjectServiceTest.java @@ -6,6 +6,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.joins.sample.SampleSequencingObjectJoinRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceConcatenationRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; @@ -30,6 +32,7 @@ public class SequencingObjectServiceTest { SampleSequencingObjectJoinRepository ssoRepository; SequenceConcatenationRepository concatenationRepository; Validator validator; + IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -37,8 +40,9 @@ public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); ssoRepository = mock(SampleSequencingObjectJoinRepository.class); concatenationRepository = mock(SequenceConcatenationRepository.class); + iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); service = new SequencingObjectServiceImpl(repository, sequenceFileRepository, ssoRepository, - concatenationRepository, validator); + concatenationRepository, validator, iridaFileStorageUtility); } @Test From 1eab223352b5b57285bcc7f724b38b24d4e6e897 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 10 Jul 2020 12:33:17 -0500 Subject: [PATCH 088/655] Updated to set iridafilestorageutility before each test --- .../model/sequenceFile/unit/Fast5ObjectTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java index 4d88a7b9583..c58942e5a3b 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/Fast5ObjectTest.java @@ -5,10 +5,15 @@ import java.nio.file.Path; import java.util.zip.GZIPOutputStream; +import org.junit.Before; import org.junit.Test; import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import static org.junit.Assert.assertEquals; /** @@ -16,6 +21,13 @@ */ public class Fast5ObjectTest { private static final String FILE_CONTENTS = "DATA CONTENTS"; + private IridaFileStorageUtility iridaFileStorageUtility; + + @Before + public void setUp() { + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + } @Test public void testCreateZippedFile() throws IOException { From 987e491117e9ca0bced011205e6e3d7e201914a3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 10 Jul 2020 17:00:10 -0500 Subject: [PATCH 089/655] Added comment to iridaFileStorageUtility bean config --- .../irida/config/services/IridaApiServicesConfig.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index bceb431d6a4..de18dbebbd5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -287,6 +287,13 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { return null; } + /** + * Used to configure both the IridaFileStorageUtility implementation + * as well as set the implementation in the IridaFiles static class + * which uses the this implementation. + * + * @return A new {@link IridaFileStorageUtility} implementation. + */ @Bean(name = "iridaFileStorageUtility") public IridaFileStorageUtility iridaFileStorageUtility() { IridaFileStorageUtility iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); From 74e0ac8eb9100f4557955a9a9cb82060ecf944a7 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 15 Jul 2020 16:16:54 -0500 Subject: [PATCH 090/655] Updated getFileSize for the aws implementation to close the s3Object after getting file size --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 9754f7a1992..95c98d43c68 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -104,9 +104,12 @@ public String getFileSize(Path file) { S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); fileSize = FileUtils.humanReadableByteCount(s3Object.getObjectMetadata() .getContentLength(), true); + s3Object.close(); } } catch (AmazonServiceException e) { logger.error("Unable to get file size from s3 bucket: " + e); + } catch (IOException e) { + logger.error("Unable to close connection to s3object: " + e); } return fileSize; } From c66fb6d34a56710858165506baef5aaf260ff07f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 15 Jul 2020 23:18:07 -0500 Subject: [PATCH 091/655] Updated tests --- .../ria/unit/web/analysis/AnalysisAjaxControllerTest.java | 3 +++ .../ria/unit/web/files/ReferenceFileControllerTest.java | 7 +++++++ .../ria/unit/web/files/SequenceFileControllerTest.java | 6 ++++++ .../irida/ria/unit/web/samples/SamplesControllerTest.java | 7 ++++++- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java index ff796e2841e..ef36d9cfe03 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisAjaxControllerTest.java @@ -44,6 +44,7 @@ import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; @@ -99,6 +100,8 @@ public void init() { httpServletResponseMock = mock(HttpServletResponse.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + analysisAjaxController = new AnalysisAjaxController(analysisSubmissionServiceMock, iridaWorkflowsServiceMock, userServiceMock, sampleService, projectServiceMock, updatePermission, metadataTemplateService, sequencingObjectService, analysisSubmissionSampleProcessor, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java index 2f17580bc47..4a46f1b179d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java @@ -30,10 +30,13 @@ import ca.corefacility.bioinformatics.irida.model.project.ProjectReferenceFileJoin; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.files.ReferenceFileController; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableList; @@ -56,12 +59,16 @@ public class ReferenceFileControllerTest { private ProjectService projectService; private ReferenceFileService referenceFileService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { projectService = mock(ProjectService.class); referenceFileService = mock(ReferenceFileService.class); messageSource = mock(MessageSource.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); // Set up the reference file Path path = Paths.get(FILE_PATH); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index 122820714b9..7e8b25e1115 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -23,10 +23,13 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Unit Tests for @{link SequenceFileController} @@ -43,12 +46,15 @@ public class SequenceFileControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; private AnalysisService analysisService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { sequencingRunService = mock(SequencingRunService.class); analysisService = mock(AnalysisService.class); objectService = mock(SequencingObjectService.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); controller = new SequenceFileController(objectService, sequencingRunService, analysisService); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index 0323eef7548..6d0d1ea2947 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -30,6 +30,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -39,6 +41,7 @@ import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -62,6 +65,7 @@ public class SamplesControllerTest { private MetadataTemplateService metadataTemplateService; private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -73,7 +77,8 @@ public void setUp() { messageSource = mock(MessageSource.class); updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); - + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); controller = new SamplesController(sampleService, projectService, sequencingObjectService, updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); } From 56ac519f9ce8049dfaf894978357291373d24e8a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 16 Jul 2020 12:45:02 -0500 Subject: [PATCH 092/655] Removed unused line of code, removed IOException from method header, removed IOException never being thrown in a try/catch block --- .../irida/ria/web/samples/SamplesAjaxController.java | 1 - .../irida/ria/web/samples/SamplesController.java | 5 +---- .../irida/service/SequencingObjectService.java | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index 79ae167df16..05dafa7f8a7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -189,7 +189,6 @@ private void createSequenceFileInSample(MultipartFile file, Sample sample) throw */ private void createFast5FileInSample(MultipartFile file, Sample sample) throws IOException { SequenceFile sequenceFile = createSequenceFile(file); - Fast5Object fast5Object = new Fast5Object(sequenceFile); sequencingObjectService.createSequencingObjectInSample(new Fast5Object(sequenceFile), sample); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 778fa534ea1..ca423798a27 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -581,10 +581,7 @@ public String concatenateSequenceFiles(@PathVariable Long sampleId, @RequestPara try { sequencingObjectService.concatenateSequences(Lists.newArrayList(readMultiple), filename, sample, removeOriginals); - } catch (IOException e) { - logger.error("Error reading files: ", e); - } - catch (ConcatenateException ex) { + } catch (ConcatenateException ex) { logger.error("Error concatenating files: ", ex); model.addAttribute("concatenateError", true); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java index 7df8dc52d28..a458feda28c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java @@ -144,5 +144,5 @@ public Map getUniqueSamplesForSequencing * @throws IOException if there was an error reading the sequences to concatenate */ public SampleSequencingObjectJoin concatenateSequences(List toJoin, String filename, - Sample targetSample, boolean removeOriginals) throws ConcatenateException, IOException; + Sample targetSample, boolean removeOriginals) throws ConcatenateException; } From 820770ee39759cbc2fba404d83a4390f4bc88fbf Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 16 Jul 2020 14:28:21 -0500 Subject: [PATCH 093/655] Removed @throws from method comment --- .../bioinformatics/irida/service/SequencingObjectService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java index a458feda28c..f9c55eb57cb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java @@ -141,7 +141,6 @@ public Map getUniqueSamplesForSequencing * the sample * @return the new {@link SampleSequencingObjectJoin} * @throws ConcatenateException if there was an error concatenating the sequences - * @throws IOException if there was an error reading the sequences to concatenate */ public SampleSequencingObjectJoin concatenateSequences(List toJoin, String filename, Sample targetSample, boolean removeOriginals) throws ConcatenateException; From 9cc8b8dc15263663fda58e5feed9acf80cbc6029 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 16 Jul 2020 14:40:21 -0500 Subject: [PATCH 094/655] Removed unused import --- .../bioinformatics/irida/service/SequencingObjectService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java index f9c55eb57cb..8eeae0ce207 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectService.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.service; -import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; From 8b92448777d5c6315b6bcf15cba595ade32246d2 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 15:32:54 -0500 Subject: [PATCH 095/655] Changed concateexception to ioexception. Updated getFileSize method in azure implementation of iridafilestorageutility to return a string --- .../IridaFileStorageAzureUtilityImpl.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index e3c850e4beb..02761c04645 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Date; @@ -20,6 +21,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -77,12 +79,12 @@ public File getTemporaryFile(Path file) { * {@inheritDoc} */ @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; + public String getFileSize(Path file) { + String fileSize = "N/A"; try { // We set the blobClient "path" to which we want to upload our file to blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - fileSize = blobClient.getProperties().getBlobSize(); + fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties().getBlobSize(), true); } catch (BlobStorageException e) { logger.trace("Couldn't calculate size as the file was not found on azure [" + e + "]"); } @@ -191,7 +193,7 @@ private String getAzureFileAbsolutePath(Path file) { * {@inheritDoc} */ @Override - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + public void appendToFile(Path target, SequenceFile file) throws IOException{ try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { @@ -199,11 +201,11 @@ public void appendToFile(Path target, SequenceFile file) throws ConcatenateExcep p += in.transferTo(p, l - p, out); } } catch (IOException e) { - throw new ConcatenateException("Could not open input file for reading", e); + throw new IOException("Could not open input file for reading", e); } } catch (IOException e) { - throw new ConcatenateException("Could not open target file for writing", e); + throw new IOException("Could not open target file for writing", e); } } @@ -211,7 +213,7 @@ public void appendToFile(Path target, SequenceFile file) throws ConcatenateExcep * {@inheritDoc} */ @Override - public String getFileExtension(List toConcatenate) throws ConcatenateException { + public String getFileExtension(List toConcatenate) throws IOException { String selectedExtension = null; for (SequencingObject object : toConcatenate) { @@ -223,7 +225,7 @@ public String getFileExtension(List toConcatenate) t .findFirst(); if (!currentExtensionOpt.isPresent()) { - throw new ConcatenateException("File extension is not valid " + fileName); + throw new IOException("File extension is not valid " + fileName); } String currentExtension = currentExtensionOpt.get(); @@ -231,7 +233,7 @@ public String getFileExtension(List toConcatenate) t if (selectedExtension == null) { selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { - throw new ConcatenateException( + throw new IOException( "Extensions of files to concatenate do not match " + currentExtension + " vs " + selectedExtension); } From 95e89d3e27aec6c6e87ee4d679ceeb40362f91c1 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 15:35:16 -0500 Subject: [PATCH 096/655] Renamed method parameter to be generic. Updated text displayed when an IOException is thrown --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 6 +++--- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 02761c04645..53cf68b5750 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -213,9 +213,9 @@ public void appendToFile(Path target, SequenceFile file) throws IOException{ * {@inheritDoc} */ @Override - public String getFileExtension(List toConcatenate) throws IOException { + public String getFileExtension(List sequencingObjects) throws IOException { String selectedExtension = null; - for (SequencingObject object : toConcatenate) { + for (SequencingObject object : sequencingObjects) { for (SequenceFile file : object.getFiles()) { String fileName = getFileName(file.getFile()); @@ -234,7 +234,7 @@ public String getFileExtension(List toConcatenate) t selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { throw new IOException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 28d0599156f..4316a094e79 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -187,7 +187,7 @@ public String getFileExtension(List sequencingObject selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { throw new IOException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } From c8583556018d5858fe0a79335181d1877154c442 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 16:19:42 -0500 Subject: [PATCH 097/655] Removed unused imports --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 53cf68b5750..8d65c8a9573 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Date; @@ -18,7 +17,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.util.FileUtils; From e91bafbbf743a246c50ac4f4acac398217beafd4 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 16:52:59 -0500 Subject: [PATCH 098/655] Updated test variable --- .../irida/ria/integration/samples/SampleDetailsPageIT.java | 2 +- .../irida/ria/integration/sequenceFiles/SequenceFilePageIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java index b46896841d9..9b1c5af3c78 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java @@ -18,7 +18,7 @@ @DatabaseSetup("/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplePagesIT.xml") public class SampleDetailsPageIT extends AbstractIridaUIITChromeDriver { private static final Long SAMPLE_ID = 1L; - private static final String SAMPLE_CREATED_DATE = "18 Jul. 2013"; + private static final String SAMPLE_CREATED_DATE = "18 Jul 2013"; private static final String SAMPLE_ORGANISM = "E. coli"; private static final String SAMPLE_LATITUDE = "49.8994"; private static final String SAMPLE_LONGITUDE = "-97.1392"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index e8410e6de88..78cd6f15327 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul. 18, 2013"; + private static final String FILE_CREATED = "Jul 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; From 0817eacadba60c4d9fb9b3a11424e384345dd07d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 17:43:10 -0500 Subject: [PATCH 099/655] Renamed method parameter to be generic. Updated text displayed when an IOException is thrown. Removed unused imports --- .../IridaFileStorageAwsUtilityImpl.java | 26 +++++++++---------- .../IridaFileStorageLocalUtilityImpl.java | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 53e34cf4e5d..2da1b7c3e2d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -14,9 +14,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -94,12 +94,12 @@ public File getTemporaryFile(Path file) { * {@inheritDoc} */ @Override - public Long getFileSize(Path file) { - Long fileSize = 0L; + public String getFileSize(Path file) { + String fileSize = "N/A"; try { S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); - fileSize = s3Object.getObjectMetadata() - .getContentLength(); + fileSize = FileUtils.humanReadableByteCount(s3Object.getObjectMetadata() + .getContentLength(), true); s3Object.close(); } catch (AmazonServiceException e) { logger.error("Unable to get file size from s3 bucket: " + e); @@ -193,7 +193,7 @@ public boolean isGzipped(Path file) throws IOException { * {@inheritDoc} */ @Override - public void appendToFile(Path target, SequenceFile file) throws ConcatenateException { + public void appendToFile(Path target, SequenceFile file) throws IOException { try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { @@ -201,11 +201,11 @@ public void appendToFile(Path target, SequenceFile file) throws ConcatenateExcep p += in.transferTo(p, l - p, out); } } catch (IOException e) { - throw new ConcatenateException("Could not open input file for reading", e); + throw new IOException("Could not open input file for reading", e); } } catch (IOException e) { - throw new ConcatenateException("Could not open target file for writing", e); + throw new IOException("Could not open target file for writing", e); } } @@ -213,9 +213,9 @@ public void appendToFile(Path target, SequenceFile file) throws ConcatenateExcep * {@inheritDoc} */ @Override - public String getFileExtension(List toConcatenate) throws ConcatenateException { + public String getFileExtension(List sequencingObjects) throws IOException { String selectedExtension = null; - for (SequencingObject object : toConcatenate) { + for (SequencingObject object : sequencingObjects) { for (SequenceFile file : object.getFiles()) { String fileName = getFileName(file.getFile()); @@ -225,7 +225,7 @@ public String getFileExtension(List toConcatenate) t .findFirst(); if (!currentExtensionOpt.isPresent()) { - throw new ConcatenateException("File extension is not valid " + fileName); + throw new IOException("File extension is not valid " + fileName); } String currentExtension = currentExtensionOpt.get(); @@ -233,8 +233,8 @@ public String getFileExtension(List toConcatenate) t if (selectedExtension == null) { selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { - throw new ConcatenateException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " + throw new IOException( + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 28d0599156f..4316a094e79 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -187,7 +187,7 @@ public String getFileExtension(List sequencingObject selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { throw new IOException( - "Extensions of files to concatenate do not match " + currentExtension + " vs " + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } From b4d32bdd85fa8a6058fd067350835757075cfad9 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 18:35:10 -0500 Subject: [PATCH 100/655] Removed unused imports and updated tests --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 2 -- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 3 --- .../irida/ria/integration/samples/SampleDetailsPageIT.java | 2 +- .../ria/integration/sequenceFiles/SequenceFilePageIT.java | 2 +- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 95c98d43c68..3ded65929b7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -14,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.util.FileUtils; @@ -27,7 +26,6 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.azure.storage.blob.models.BlobStorageException; /** * Component implementation of file utitlities for aws storage diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 352291a4571..abe7306f738 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -6,10 +6,8 @@ import java.io.InputStream; import java.nio.channels.FileChannel; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Date; import java.util.List; import java.util.Optional; import java.util.zip.GZIPInputStream; @@ -19,7 +17,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.util.FileUtils; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java index b46896841d9..9b1c5af3c78 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/samples/SampleDetailsPageIT.java @@ -18,7 +18,7 @@ @DatabaseSetup("/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplePagesIT.xml") public class SampleDetailsPageIT extends AbstractIridaUIITChromeDriver { private static final Long SAMPLE_ID = 1L; - private static final String SAMPLE_CREATED_DATE = "18 Jul. 2013"; + private static final String SAMPLE_CREATED_DATE = "18 Jul 2013"; private static final String SAMPLE_ORGANISM = "E. coli"; private static final String SAMPLE_LATITUDE = "49.8994"; private static final String SAMPLE_LONGITUDE = "-97.1392"; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java index e8410e6de88..78cd6f15327 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/sequenceFiles/SequenceFilePageIT.java @@ -20,7 +20,7 @@ public class SequenceFilePageIT extends AbstractIridaUIITChromeDriver { private static final String FILE_NAME = "test_file.fastq"; private static final String FILE_ID = "1"; private static final String FILE_ENCODING = "Sanger / Illumina 1.9"; - private static final String FILE_CREATED = "Jul. 18, 2013"; + private static final String FILE_CREATED = "Jul 18, 2013"; private static final String FILE_TOTAL_SEQUENCE = "4"; private static final String FILE_TOTAL_BASES = "937"; private static final String FILE_MIN_LENGTH = "184"; From 323b144a908bb2b931396c89d58f126867bc0068 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 20 Jul 2020 21:33:56 -0500 Subject: [PATCH 101/655] Updated tests to set iridaFileStorageUtility in IridaFiles before running tests --- .../ria/unit/web/files/ReferenceFileControllerTest.java | 7 ++++++- .../ria/unit/web/files/SequenceFileControllerTest.java | 6 ++++++ .../irida/ria/unit/web/samples/SamplesControllerTest.java | 6 ++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java index 1619fed8fc0..b46a882ca43 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileControllerTest.java @@ -29,10 +29,13 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ProjectReferenceFileJoin; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.files.ReferenceFileController; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableList; @@ -55,19 +58,21 @@ public class ReferenceFileControllerTest { private ProjectService projectService; private ReferenceFileService referenceFileService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { projectService = mock(ProjectService.class); referenceFileService = mock(ReferenceFileService.class); messageSource = mock(MessageSource.class); - + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); // Set up the reference file Path path = Paths.get(FILE_PATH); ReferenceFile file = new ReferenceFile(path); when(referenceFileService.read(FILE_ID)).thenReturn(file); controller = new ReferenceFileController(projectService, referenceFileService, messageSource); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @Test diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index 3b844b36398..bf4ea948f77 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -22,10 +22,13 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Unit Tests for @{link SequenceFileController} @@ -42,13 +45,16 @@ public class SequenceFileControllerTest { private SequencingRunService sequencingRunService; private SequencingObjectService objectService; private AnalysisService analysisService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { sequencingRunService = mock(SequencingRunService.class); analysisService = mock(AnalysisService.class); objectService = mock(SequencingObjectService.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); controller = new SequenceFileController(objectService, sequencingRunService, analysisService); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); Path path = Paths.get(FILE_PATH); SequenceFile file = new SequenceFile(path); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java index 0d80ea2096e..98e153521af 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesControllerTest.java @@ -30,6 +30,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesController; import ca.corefacility.bioinformatics.irida.security.permissions.sample.ReadSamplePermission; @@ -39,6 +41,7 @@ import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -62,6 +65,7 @@ public class SamplesControllerTest { private MetadataTemplateService metadataTemplateService; private GenomeAssemblyService genomeAssemblyService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { @@ -73,8 +77,10 @@ public void setUp() { messageSource = mock(MessageSource.class); updateSamplePermission = mock(UpdateSamplePermission.class); readSamplePermission = mock(ReadSamplePermission.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); controller = new SamplesController(sampleService, projectService, sequencingObjectService, updateSamplePermission, metadataTemplateService, genomeAssemblyService, messageSource); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } // ************************************************************************************************ From fbf62761e1a6aeec10148cdc33de80b316cfa270 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 21 Jul 2020 10:49:39 -0500 Subject: [PATCH 102/655] Removed unused imports --- .../irida/model/workflow/analysis/AnalysisOutputFile.java | 1 - .../irida/ria/web/analysis/AnalysisAjaxController.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index d96e7722d97..acbf4bc53d5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Date; import java.util.Objects; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 6d637a8ff12..8f10c6bb97d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.ria.web.analysis; import java.io.*; -import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.security.Principal; From d3b67f0e8175866cb8e7fee9569a611e1f2056cd Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 21 Jul 2020 13:56:27 -0500 Subject: [PATCH 103/655] Removed objectmapper to write byte files and replaced with iridafilestorageutility method --- .../bioinformatics/irida/ria/utilities/FileUtilities.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java index 82ee4125e0b..b690644bb1c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java @@ -14,7 +14,6 @@ import org.apache.commons.io.IOUtils; import org.apache.poi.ss.usermodel.*; -import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,8 +88,7 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re // the next entry. outputStream.closeEntry(); - ObjectMapper objectMapper = new ObjectMapper(); - byte[] bytes = objectMapper.writeValueAsBytes(file); + byte[] bytes = file.getBytesForFile(); outputStream.putNextEntry(new ZipEntry(zipEntryName.toString() + "-prov.json")); outputStream.write(bytes); outputStream.closeEntry(); From 6328a2751ba469be45ce8220673516bc645f448f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 21 Jul 2020 17:00:20 -0500 Subject: [PATCH 104/655] Added default null for galaxy.cloud.storage.temporary.directory if not set in conf file --- .../irida/config/services/IridaApiServicesConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index b8eca299698..b2a4ea8e1e9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -150,7 +150,7 @@ public class IridaApiServicesConfig { @Value("${aws.secret.key:#{null}}") private String awsSecretKey; - @Value("${galaxy.cloud.storage.temporary.directory}") + @Value("${galaxy.cloud.storage.temporary.directory:#{null}}") private String cloudStorageTemporaryDirectory; From 7cda1785b63cda8e7048d0548e52dc537407b21a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 23 Jul 2020 16:41:23 -0500 Subject: [PATCH 105/655] Added temporary file cleanup (files downloaded from an object store). Changed getTemporaryFile method to getFile --- .../upload/galaxy/GalaxyHistoriesService.java | 9 +++- .../upload/galaxy/GalaxyLibrariesService.java | 2 +- .../processing/impl/FastqcFileProcessor.java | 14 +++++- .../IridaFileStorageAwsUtilityImpl.java | 4 +- .../IridaFileStorageAzureUtilityImpl.java | 4 +- .../IridaFileStorageLocalUtilityImpl.java | 2 +- .../filesystem/IridaFileStorageUtility.java | 4 +- .../web/analysis/AnalysisAjaxController.java | 18 +++++-- .../ria/web/dto/GalaxyUploadDetails.java | 49 +++++++++++++++++++ .../bioinformatics/irida/util/FileUtils.java | 25 ++++++++++ 10 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java index c3568dc59ab..23ed3e65f7a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java @@ -30,6 +30,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.github.jmchilton.blend4j.galaxy.ToolsClient; @@ -155,7 +156,7 @@ public Dataset fileToHistory(Path path, InputFileType fileType, History history) checkNotNull(history.getId(), "history id is null"); checkState(iridaFileStorageUtility.fileExists(path), "path " + path + " does not exist"); - File file = iridaFileStorageUtility.getTemporaryFile(path); + File file = iridaFileStorageUtility.getFile(path); FileUploadRequest uploadRequest = new FileUploadRequest(history.getId(), file); uploadRequest.setFileType(fileType.toString()); @@ -175,7 +176,11 @@ public Dataset fileToHistory(Path path, InputFileType fileType, History history) throw new UploadException(message); } else { - return getDatasetForFileInHistory(file.getName(), history.getId()); + Dataset dataset = getDatasetForFileInHistory(file.getName(), history.getId()); + if(!iridaFileStorageUtility.storageTypeIsLocal()) { + FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + } + return dataset; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index 4eb644a45c8..dbdad35d83c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -155,7 +155,7 @@ public String fileToLibrary(Path path, InputFileType fileType, checkNotNull(library.getId(), "library id is null"); checkState(iridaFileStorageUtility.fileExists(path), "path " + path + " does not exist"); - File file = iridaFileStorageUtility.getTemporaryFile(path); + File file = iridaFileStorageUtility.getFile(path); try { LibraryContent rootContent = librariesClient.getRootFolder(library diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index b8c20202d54..3f1f71e6f73 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -12,6 +12,8 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.util.FileUtils; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,10 +34,12 @@ import javax.imageio.ImageIO; import java.awt.*; import java.awt.image.BufferedImage; +import java.io.File; import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -96,8 +100,8 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx .description(messageSource.getMessage("fastqc.file.processor.analysis.description", new Object[] {FastQCApplication.VERSION}, LocaleContextHolder.getLocale())); try { - uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - iridaFileStorageUtility.getTemporaryFile(fileToProcess)); + File fastQCSequenceFileToProcess = iridaFileStorageUtility.getFile(fileToProcess); + uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile(fastQCSequenceFileToProcess); BasicStats basicStats = new BasicStats(); PerBaseQualityScores pbqs = new PerBaseQualityScores(); PerSequenceQualityScores psqs = new PerSequenceQualityScores(); @@ -130,6 +134,12 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx sequenceFile.setFastQCAnalysis(analysis.build()); sequenceFileRepository.saveMetadata(sequenceFile); + + if(!iridaFileStorageUtility.storageTypeIsLocal()) { + // Delete the temporarily downloaded sequence file as it + // has been processed and it's parent directory + FileUtils.removeTemporaryFile(fastQCSequenceFileToProcess.getParent(), fastQCSequenceFileToProcess.toPath()); + } } catch (Exception e) { logger.error("FastQC failed to process the sequence file: " + e.getMessage()); throw new FileProcessorException("FastQC failed to parse the sequence file.", e); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 60df0202bd2..fad66724771 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -63,7 +63,7 @@ public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, St * {@inheritDoc} */ @Override - public File getTemporaryFile(Path file) { + public File getFile(Path file) { File fileToProcess = null; try { @@ -207,7 +207,7 @@ public boolean isGzipped(Path file) throws IOException { public void appendToFile(Path target, SequenceFile file) throws IOException { try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { + try (FileChannel in = new FileInputStream(getFile(file.getFile())).getChannel()) { for (long p = 0, l = in.size(); p < l; ) { p += in.transferTo(p, l - p, out); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index a453667ad0c..090989a91a9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -66,7 +66,7 @@ public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerNa * {@inheritDoc} */ @Override - public File getTemporaryFile(Path file) { + public File getFile(Path file) { File fileToProcess = null; // We set the blobClient "path" to which we want to upload our file to @@ -206,7 +206,7 @@ public boolean isGzipped(Path file) throws IOException { public void appendToFile(Path target, SequenceFile file) throws IOException { try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { + try (FileChannel in = new FileInputStream(getFile(file.getFile())).getChannel()) { for (long p = 0, l = in.size(); p < l; ) { p += in.transferTo(p, l - p, out); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 9eca54a9d8b..12825bc9ce4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -38,7 +38,7 @@ public IridaFileStorageLocalUtilityImpl(){ * {@inheritDoc} */ @Override - public File getTemporaryFile(Path file) { + public File getFile(Path file) { File fileToProcess = null; fileToProcess = file.toFile(); return fileToProcess; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 478f21a298e..06f073487da 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -19,12 +19,12 @@ public interface IridaFileStorageUtility { //Valid extensions to try to concatenate with this tool public static final List VALID_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** - * Get a temporarry file from storage + * Get a file from storage * * @param file The {@link Path} to the file * @return {@link File} which was retrieved from path */ - public File getTemporaryFile(Path file); + public File getFile(Path file); /** * Get file size diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index c0cf985ae83..4408c4c252a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -59,6 +59,7 @@ import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; import ca.corefacility.bioinformatics.irida.model.project.Project; +import ca.corefacility.bioinformatics.irida.util.FileUtils; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.fasterxml.jackson.core.JsonParseException; @@ -446,9 +447,13 @@ private void addFirstLine(AnalysisOutputFileInfo info, AnalysisOutputFile aof) { RandomAccessFile reader = null; final Path aofFile = aof.getFile(); try { - reader = new RandomAccessFile(iridaFileStorageUtility.getTemporaryFile(aofFile), "r"); + File file = iridaFileStorageUtility.getFile(aofFile); + reader = new RandomAccessFile(file, "r"); info.setFirstLine(reader.readLine()); info.setFilePointer(reader.getFilePointer()); + if(!iridaFileStorageUtility.storageTypeIsLocal()) { + FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + } } catch (FileNotFoundException e) { logger.error("Could not find file '" + aofFile + "' " + e); } catch (IOException e) { @@ -506,7 +511,7 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable contents.setToolName(tool.getToolName()); contents.setToolVersion(tool.getToolVersion()); try { - final File file = iridaFileStorageUtility.getTemporaryFile(aofFile); + final File file = iridaFileStorageUtility.getFile(aofFile); final RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r"); randomAccessFile.seek(seek); if (seek == 0) { @@ -536,6 +541,9 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable } } contents.setFilePointer(randomAccessFile.getFilePointer()); + if(!iridaFileStorageUtility.storageTypeIsLocal()) { + FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + } } catch (IOException e) { logger.error("Could not read output file '" + aof.getId() + "' " + e); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); @@ -746,7 +754,8 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { .getFile(); try { - String json = new Scanner(new BufferedReader(new FileReader(iridaFileStorageUtility.getTemporaryFile(path)))).useDelimiter("\\Z") + File file = iridaFileStorageUtility.getFile(path); + String json = new Scanner(new BufferedReader(new FileReader(file))).useDelimiter("\\Z") .next(); // verify file is proper json file and map to a SistrResult list @@ -759,6 +768,9 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { if (samples != null && samples.size() == 1) { Sample sample = samples.iterator() .next(); + if(!iridaFileStorageUtility.storageTypeIsLocal()) { + FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + } return new AnalysisSistrResults(sample.getSampleName(), false, sistrResults.get(0)); } else { logger.error("Invalid number of associated samples for submission " + submission); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java new file mode 100644 index 00000000000..e48be020dc4 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java @@ -0,0 +1,49 @@ +package ca.corefacility.bioinformatics.irida.ria.web.dto; + +import java.nio.file.Path; + +public class GalaxyUploadDetails { + private String datasetLibraryId; + private Path path; + private String dirToRemove; + private Path fileToRemove; + + public GalaxyUploadDetails(String datasetLibraryId, Path path, String dirToRemove, Path fileToRemove) { + this.datasetLibraryId = datasetLibraryId; + this.path = path; + this.dirToRemove = dirToRemove; + this.fileToRemove = fileToRemove; + } + + public String getDatasetLibraryId() { + return datasetLibraryId; + } + + public void setDatasetLibraryId(String datasetLibraryId) { + this.datasetLibraryId = datasetLibraryId; + } + + public Path getPath() { + return path; + } + + public void setPath(Path path) { + this.path = path; + } + + public Path getFileToRemove() { + return fileToRemove; + } + + public void setFileToRemove(Path fileToRemove) { + this.fileToRemove = fileToRemove; + } + + public String getDirToRemove() { + return dirToRemove; + } + + public void setDirToRemove(String dirToRemove) { + this.dirToRemove = dirToRemove; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index b4a8b915bee..6384f8bae7d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.zip.GZIPInputStream; @@ -47,4 +48,28 @@ public static String humanReadableByteCount(long bytes, boolean si) { String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); } + + /** + * Removes temporarily downloaded files from an object store off the local filesystem + * + * @param parentDirectoryPath The parent directory. + * @param fileToRemove The file to remove + */ + public static void removeTemporaryFile(String parentDirectoryPath, Path fileToRemove){ + try { + Path dirToRemovePath = Paths.get(parentDirectoryPath); + try { + Files.delete(fileToRemove); + } catch (IOException e) { + throw new IOException("Unable to find file to remove " + e); + } + try { + Files.delete(dirToRemovePath); + } catch (IOException e) { + throw new IOException("Unable to find parent directory to remove " + e); + } + } catch(IOException e) { + + } + } } From c289104ccae4b733d9e11be21c0ac9b74c11db83 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 23 Jul 2020 17:01:55 -0500 Subject: [PATCH 106/655] Removed dto not used. Removed unused imports --- .../processing/impl/FastqcFileProcessor.java | 2 - .../ria/web/dto/GalaxyUploadDetails.java | 49 ------------------- 2 files changed, 51 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 3f1f71e6f73..f0a5784a7ab 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -13,7 +13,6 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.util.FileUtils; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +38,6 @@ import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collections; import java.util.HashSet; import java.util.Set; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java deleted file mode 100644 index e48be020dc4..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/GalaxyUploadDetails.java +++ /dev/null @@ -1,49 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.dto; - -import java.nio.file.Path; - -public class GalaxyUploadDetails { - private String datasetLibraryId; - private Path path; - private String dirToRemove; - private Path fileToRemove; - - public GalaxyUploadDetails(String datasetLibraryId, Path path, String dirToRemove, Path fileToRemove) { - this.datasetLibraryId = datasetLibraryId; - this.path = path; - this.dirToRemove = dirToRemove; - this.fileToRemove = fileToRemove; - } - - public String getDatasetLibraryId() { - return datasetLibraryId; - } - - public void setDatasetLibraryId(String datasetLibraryId) { - this.datasetLibraryId = datasetLibraryId; - } - - public Path getPath() { - return path; - } - - public void setPath(Path path) { - this.path = path; - } - - public Path getFileToRemove() { - return fileToRemove; - } - - public void setFileToRemove(Path fileToRemove) { - this.fileToRemove = fileToRemove; - } - - public String getDirToRemove() { - return dirToRemove; - } - - public void setDirToRemove(String dirToRemove) { - this.dirToRemove = dirToRemove; - } -} From 91132c3086ca321219e35651d97fc316ab7d7447 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 24 Jul 2020 14:13:34 -0500 Subject: [PATCH 107/655] Removed use of another directory for temp downloads and made use of existing directories. Added file cleanup after an analysis has completed or errored when using an object store --- .../services/IridaApiServicesConfig.java | 8 ++--- .../AnalysisScheduledTaskConfig.java | 11 +++++- .../upload/galaxy/GalaxyHistoriesService.java | 3 -- .../processing/impl/FastqcFileProcessor.java | 2 +- .../IridaFileStorageAwsUtilityImpl.java | 28 ++------------- .../IridaFileStorageAzureUtilityImpl.java | 31 ++-------------- .../web/analysis/AnalysisAjaxController.java | 6 ++-- .../AnalysisExecutionScheduledTaskImpl.java | 36 +++++++++++++++++-- .../bioinformatics/irida/util/FileUtils.java | 33 +++++++++-------- .../irida/config/filesystem.properties | 2 -- .../integration/SNVPhylAnalysisIT.java | 8 ++++- .../integration/GalaxyJobErrorsServiceIT.java | 15 +++++--- .../AnalysisExecutionScheduledTaskImplIT.java | 17 +++++---- ...nalysisExecutionScheduledTaskImplTest.java | 20 +++++++---- 14 files changed, 114 insertions(+), 106 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index b2a4ea8e1e9..4d717a335e7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -150,10 +150,6 @@ public class IridaApiServicesConfig { @Value("${aws.secret.key:#{null}}") private String awsSecretKey; - @Value("${galaxy.cloud.storage.temporary.directory:#{null}}") - private String cloudStorageTemporaryDirectory; - - @Autowired private IridaPluginConfig.IridaPluginList pipelinePlugins; @@ -322,9 +318,9 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { public IridaFileStorageUtility iridaFileStorageUtility() { IridaFileStorageUtility iridaFileStorageUtility; if(storageType.equalsIgnoreCase("azure")) { - iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(connectionStr, containerName, cloudStorageTemporaryDirectory); + iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(connectionStr, containerName); } else if (storageType.equalsIgnoreCase("aws")) { - iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey, cloudStorageTemporaryDirectory); + iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); } else { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java index c680a3ffc17..9654090a285 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java @@ -3,8 +3,10 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; +import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.impl.AnalysisExecutionScheduledTaskImpl; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.CleanupAnalysisSubmissionConditionAge; @@ -46,6 +48,12 @@ public class AnalysisScheduledTaskConfig { @Autowired private EmailController emailController; + @Autowired + private SequencingObjectService sequencingObjectService; + + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + /** * Defines the time to clean up in number of days a submission must exist before it is cleaned up. */ @@ -120,7 +128,8 @@ public void cleanupAnalysisSubmissions() { @Bean public AnalysisExecutionScheduledTask analysisExecutionScheduledTask() { return new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, - cleanupAnalysisSubmissionCondition(), galaxyJobErrorsService, jobErrorRepository, emailController); + cleanupAnalysisSubmissionCondition(), galaxyJobErrorsService, jobErrorRepository, emailController, + sequencingObjectService, iridaFileStorageUtility); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java index 23ed3e65f7a..5e7a825036a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java @@ -177,9 +177,6 @@ public Dataset fileToHistory(Path path, InputFileType fileType, History history) throw new UploadException(message); } else { Dataset dataset = getDatasetForFileInHistory(file.getName(), history.getId()); - if(!iridaFileStorageUtility.storageTypeIsLocal()) { - FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); - } return dataset; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index f0a5784a7ab..0ea7a1623a6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -136,7 +136,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx if(!iridaFileStorageUtility.storageTypeIsLocal()) { // Delete the temporarily downloaded sequence file as it // has been processed and it's parent directory - FileUtils.removeTemporaryFile(fastQCSequenceFileToProcess.getParent(), fastQCSequenceFileToProcess.toPath()); + FileUtils.removeTemporaryFile(fastQCSequenceFileToProcess.toPath()); } } catch (Exception e) { logger.error("FastQC failed to process the sequence file: " + e.getMessage()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index fad66724771..93c9cd05249 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -2,23 +2,16 @@ import java.io.*; import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.zip.GZIPInputStream; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -41,22 +34,16 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsUtilityImpl.class); - @Value("${galaxy.tempfile.directory.permissions}") - private String filePermissions; - private String bucketName; private BasicAWSCredentials awsCreds; private AmazonS3 s3; - private String tempDir; - @Autowired - public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey, String cloudStorageTemporaryDirectory){ + public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey){ this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); this.s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion)) .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); this.bucketName = bucketName; - this.tempDir = cloudStorageTemporaryDirectory; } /** @@ -67,22 +54,11 @@ public File getFile(Path file) { File fileToProcess = null; try { - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String [] nameTokens = file.toAbsolutePath().toString().split("/"); - String fileName = nameTokens[nameTokens.length-1]; - S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); - FileAttribute> fileAttributes = PosixFilePermissions - .asFileAttribute(PosixFilePermissions.fromString(filePermissions)); - - Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "aws-temp-file-", fileAttributes); - Path target = targetDirectory.resolve(fileName); - // Copy the the file from the bucket into a local file - File targetFile = new File(target.toAbsolutePath().toString()); + File targetFile = new File(file.toAbsolutePath().toString()); FileUtils.copyInputStreamToFile(s3ObjectInputStream, targetFile); fileToProcess = targetFile; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 090989a91a9..07ef921d378 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -5,26 +5,18 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; - import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.zip.GZIPInputStream; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -44,22 +36,16 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureUtilityImpl.class); - @Value("${galaxy.tempfile.directory.permissions}") - private String filePermissions; - private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient ; private BlobClient blobClient; - private String tempDir; - @Autowired - public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerName, String cloudStorageTemporaryDirectory){ + public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerName){ this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - this.tempDir = cloudStorageTemporaryDirectory; } /** @@ -73,22 +59,9 @@ public File getFile(Path file) { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - - String [] blobNameTokens = blobClient.getBlobName().split("/"); - String fileName = blobNameTokens[blobNameTokens.length-1]; - - FileAttribute> fileAttributes = PosixFilePermissions - .asFileAttribute(PosixFilePermissions.fromString(filePermissions)); - - Path targetDirectory = Files.createTempDirectory(Paths.get(tempDir), "azure-temp-file-", fileAttributes); - Path target = targetDirectory.resolve(fileName); - InputStream initialStream = blobClient.openInputStream(); - File targetFile = new File(target.toAbsolutePath().toString()); + File targetFile = new File(file.toAbsolutePath().toString()); FileUtils.copyInputStreamToFile(initialStream, targetFile); - fileToProcess = targetFile; } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 4408c4c252a..073c045ae64 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -452,7 +452,7 @@ private void addFirstLine(AnalysisOutputFileInfo info, AnalysisOutputFile aof) { info.setFirstLine(reader.readLine()); info.setFilePointer(reader.getFilePointer()); if(!iridaFileStorageUtility.storageTypeIsLocal()) { - FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + FileUtils.removeTemporaryFile(file.toPath()); } } catch (FileNotFoundException e) { logger.error("Could not find file '" + aofFile + "' " + e); @@ -542,7 +542,7 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable } contents.setFilePointer(randomAccessFile.getFilePointer()); if(!iridaFileStorageUtility.storageTypeIsLocal()) { - FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + FileUtils.removeTemporaryFile(file.toPath()); } } catch (IOException e) { logger.error("Could not read output file '" + aof.getId() + "' " + e); @@ -769,7 +769,7 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { Sample sample = samples.iterator() .next(); if(!iridaFileStorageUtility.storageTypeIsLocal()) { - FileUtils.removeTemporaryFile(file.getParent(), file.toPath()); + FileUtils.removeTemporaryFile(file.toPath()); } return new AnalysisSistrResults(sample.getSampleName(), false, sistrResults.get(0)); } else { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index c7d9da90a67..0f48d2986f6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -15,6 +15,9 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; @@ -22,10 +25,13 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; +import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.EmailController; +import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.google.common.collect.Sets; @@ -51,6 +57,8 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche private GalaxyJobErrorsService galaxyJobErrorsService; private JobErrorRepository jobErrorRepository; private final EmailController emailController; + private SequencingObjectService sequencingObjectService; + private IridaFileStorageUtility iridaFileStorageUtility; /** * Builds a new AnalysisExecutionScheduledTaskImpl with the given service @@ -63,18 +71,22 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche * @param galaxyJobErrorsService {@link GalaxyJobErrorsService} for getting {@link JobError} objects * @param jobErrorRepository {@link JobErrorRepository} for {@link JobError} objects * @param emailController {@link EmailController} for sending completion/error emails for {@link AnalysisSubmission}s + * @param sequencingObjectService {@link SequencingObjectService} for getting the {@link SequencingObject}s + * @param iridaFileStorageUtility The irida file storage utility implementation */ @Autowired public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisSubmissionRepository, AnalysisExecutionService analysisExecutionServiceGalaxy, CleanupAnalysisSubmissionCondition cleanupCondition, GalaxyJobErrorsService galaxyJobErrorsService, - JobErrorRepository jobErrorRepository, EmailController emailController) { + JobErrorRepository jobErrorRepository, EmailController emailController, SequencingObjectService sequencingObjectService, IridaFileStorageUtility iridaFileStorageUtility) { this.analysisSubmissionRepository = analysisSubmissionRepository; this.analysisExecutionService = analysisExecutionServiceGalaxy; this.cleanupCondition = cleanupCondition; this.galaxyJobErrorsService = galaxyJobErrorsService; this.jobErrorRepository = jobErrorRepository; this.emailController = emailController; + this.sequencingObjectService = sequencingObjectService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -298,8 +310,26 @@ private Future handleWorkflowStatus(GalaxyWorkflowStatus wor and the user selected to be emailed on completion, then the following code will be executed. */ - if (finalWorkflowStatusSet && analysisSubmission.getEmailPipelineResult()) { - emailController.sendPipelineStatusEmail(analysisSubmission); + if (finalWorkflowStatusSet) { + if(analysisSubmission.getEmailPipelineResult()) { + emailController.sendPipelineStatusEmail(analysisSubmission); + } + + // Cleanup files downloaded from object store + if(!iridaFileStorageUtility.storageTypeIsLocal()) { + Set inputFilePairs = sequencingObjectService.getSequencingObjectsOfTypeForAnalysisSubmission( + analysisSubmission, SequenceFilePair.class); + Set inputFilesSingle = sequencingObjectService.getSequencingObjectsOfTypeForAnalysisSubmission( + analysisSubmission, SingleEndSequenceFile.class); + for(SequenceFilePair sfp : inputFilePairs) { + for(SequenceFile sf : sfp.getFiles()) { + FileUtils.removeTemporaryFile(sf.getFile()); + } + } + for(SingleEndSequenceFile ssf : inputFilesSingle) { + FileUtils.removeTemporaryFile(ssf.getSequenceFile().getFile()); + } + } } return returnedSubmission; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index 6384f8bae7d..d522d7e458b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -8,10 +8,14 @@ import java.nio.file.StandardOpenOption; import java.util.zip.GZIPInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * A class containing a number of utilities for dealing with files. */ public class FileUtils { + private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); /** * Determines if a file is compressed. Adapted from stackoverflow answer: @@ -51,25 +55,26 @@ public static String humanReadableByteCount(long bytes, boolean si) { /** * Removes temporarily downloaded files from an object store off the local filesystem + * and the parent directory if it is empty * - * @param parentDirectoryPath The parent directory. * @param fileToRemove The file to remove */ - public static void removeTemporaryFile(String parentDirectoryPath, Path fileToRemove){ + public static void removeTemporaryFile(Path fileToRemove){ + + String parentDirectoryPath = fileToRemove.getParent().toString(); + Path dirToRemovePath = Paths.get(parentDirectoryPath); + try { - Path dirToRemovePath = Paths.get(parentDirectoryPath); - try { - Files.delete(fileToRemove); - } catch (IOException e) { - throw new IOException("Unable to find file to remove " + e); - } - try { - Files.delete(dirToRemovePath); - } catch (IOException e) { - throw new IOException("Unable to find parent directory to remove " + e); - } - } catch(IOException e) { + Files.delete(fileToRemove); + } catch (IOException e) { + logger.debug("Unable to find file to remove " + e); + } + try { + org.apache.commons.io.FileUtils.deleteDirectory(dirToRemovePath.toFile()); + } catch (IOException e) { + logger.debug("Unable to remove directory " + e); } } + } diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties index 436e1b0d588..1b8a047b91f 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/config/filesystem.properties @@ -13,5 +13,3 @@ file.processing.process=true file.upload.max_size=21474836480 irida.storage.type=local - -galaxy.tempfile.directory.permissions=rwxrwxrwx \ No newline at end of file diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java index 55c8458c514..481795aff63 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java @@ -59,6 +59,7 @@ import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; @@ -125,6 +126,10 @@ public class SNVPhylAnalysisIT { private AnalysisExecutionScheduledTask analysisExecutionScheduledTask; + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + + private Path sequenceFilePathA1; private Path sequenceFilePathA2; private Path sequenceFilePathB1; @@ -169,7 +174,8 @@ public void setup() throws URISyntaxException, IOException { Assume.assumeFalse(WindowsPlatformCondition.isWindows()); analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, - analysisExecutionService, CleanupAnalysisSubmissionCondition.NEVER_CLEANUP, galaxyJobErrorsService, jobErrorRepository, emailController); + analysisExecutionService, CleanupAnalysisSubmissionCondition.NEVER_CLEANUP, galaxyJobErrorsService, + jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); Path tempDir = Files.createTempDirectory(rootTempDirectory, "snvphylTest"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java index 8aac265edbd..889cc173ab9 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java @@ -30,12 +30,10 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; -import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; -import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; -import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.service.*; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.impl.AnalysisExecutionScheduledTaskImpl; -import ca.corefacility.bioinformatics.irida.service.EmailController; import com.github.springtestdbunit.DbUnitTestExecutionListener; import com.github.springtestdbunit.annotation.DatabaseSetup; @@ -79,6 +77,13 @@ public class GalaxyJobErrorsServiceIT { @Autowired private EmailController emailController; + @Autowired + private SequencingObjectService sequencingObjectService; + + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + + @Before public void setup() throws URISyntaxException, IOException { Assume.assumeFalse(WindowsPlatformCondition.isWindows()); @@ -97,7 +102,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController); + jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java index 9dc1d9621f9..db9ec08d649 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java @@ -41,14 +41,12 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.user.UserRepository; -import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; -import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; -import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; +import ca.corefacility.bioinformatics.irida.service.*; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.impl.AnalysisExecutionScheduledTaskImpl; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.CleanupAnalysisSubmissionConditionAge; -import ca.corefacility.bioinformatics.irida.service.EmailController; import com.github.springtestdbunit.DbUnitTestExecutionListener; import com.github.springtestdbunit.annotation.DatabaseSetup; @@ -92,6 +90,13 @@ public class AnalysisExecutionScheduledTaskImplIT { private AnalysisExecutionScheduledTask analysisExecutionScheduledTask; + @Autowired + private SequencingObjectService sequencingObjectService; + + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + + private Path sequenceFilePath; private Path sequenceFilePath2; private Path referenceFilePath; @@ -115,7 +120,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, - galaxyJobErrorsService, jobErrorRepository, emailController); + galaxyJobErrorsService, jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); Path sequenceFilePathReal = Paths .get(DatabaseSetupGalaxyITService.class.getResource("testData1.fastq").toURI()); @@ -182,7 +187,7 @@ public void testFullAnalysisRunSuccessWithSampleUpdates() throws Exception { public void testFullAnalysisRunSuccessNoCleanupAge() throws Exception { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController); + galaxyJobErrorsService, jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); AnalysisSubmission analysisSubmission = analysisExecutionGalaxyITService.setupSubmissionInDatabase(1L, sequenceFilePath, referenceFilePath, validIridaWorkflowId, false); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index 1aba93125a1..7abb29a9d42 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -37,9 +37,12 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.Util; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; +import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.impl.AnalysisExecutionScheduledTaskImpl; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.CleanupAnalysisSubmissionConditionAge; @@ -84,6 +87,9 @@ public class AnalysisExecutionScheduledTaskImplTest { @Mock private TestEmailController emailController; + @Mock + private SequencingObjectService sequencingObjectService; + private static final String ANALYSIS_ID = "1"; private static final Long INTERNAL_ID = 1L; private AnalysisSubmission analysisSubmission; @@ -92,16 +98,18 @@ public class AnalysisExecutionScheduledTaskImplTest { private UUID workflowId = UUID.randomUUID(); + private IridaFileStorageUtility iridaFileStorageUtility; + /** * Sets up variables for tests. */ @Before public void setup() { MockitoAnnotations.initMocks(this); - + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController); + jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); analysisSubmission = AnalysisSubmission.builder(workflowId) .name("my analysis") @@ -631,7 +639,7 @@ public void testCleanupAnalysisSubmissionsCompletedCleanedCleaningSuccess() thro public void testCleanupAnalysisSubmissionsCompletedOverOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController); + galaxyJobErrorsService, jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -658,7 +666,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedCleanupZeroSuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZERO), - galaxyJobErrorsService, jobErrorRepository, emailController); + galaxyJobErrorsService, jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -685,7 +693,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZER public void testCleanupAnalysisSubmissionsCompletedUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController); + galaxyJobErrorsService, jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -712,7 +720,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedOverUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController); + galaxyJobErrorsService, jobErrorRepository, emailController, sequencingObjectService, iridaFileStorageUtility); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); From 0944704432457869ffdc57429c4f4d162f574687 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 24 Jul 2020 17:46:08 -0500 Subject: [PATCH 108/655] Fixed reference in @param --- .../irida/service/impl/AnalysisExecutionScheduledTaskImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index 0f48d2986f6..26f417bf019 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -71,7 +71,7 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche * @param galaxyJobErrorsService {@link GalaxyJobErrorsService} for getting {@link JobError} objects * @param jobErrorRepository {@link JobErrorRepository} for {@link JobError} objects * @param emailController {@link EmailController} for sending completion/error emails for {@link AnalysisSubmission}s - * @param sequencingObjectService {@link SequencingObjectService} for getting the {@link SequencingObject}s + * @param sequencingObjectService {@link SequencingObjectService} for getting the sequencing objects * @param iridaFileStorageUtility The irida file storage utility implementation */ @Autowired From fd0b6c74049b901dff398673a26555bb545aaa5d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 24 Jul 2020 18:32:52 -0500 Subject: [PATCH 109/655] Updated logic to remove files downloaded from object store and its parent directory --- .../bioinformatics/irida/util/FileUtils.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index d522d7e458b..74ce8037a11 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -64,16 +64,25 @@ public static void removeTemporaryFile(Path fileToRemove){ String parentDirectoryPath = fileToRemove.getParent().toString(); Path dirToRemovePath = Paths.get(parentDirectoryPath); - try { - Files.delete(fileToRemove); - } catch (IOException e) { - logger.debug("Unable to find file to remove " + e); + if(fileToRemove.toFile().isFile()) { + try { + Files.delete(fileToRemove); + } catch (IOException e) { + logger.debug("Unable to find file to remove " + e); + } } - try { - org.apache.commons.io.FileUtils.deleteDirectory(dirToRemovePath.toFile()); - } catch (IOException e) { - logger.debug("Unable to remove directory " + e); + // Directory is empty + if(dirToRemovePath.toFile().listFiles().length == 0) { + try { + org.apache.commons.io.FileUtils.deleteDirectory(dirToRemovePath.toFile()); + Path tDir = dirToRemovePath.toFile().toPath(); + if(tDir.toString().substring(tDir.toString().length() - 1).matches("\\d+")) { + removeTemporaryFile(tDir); + } + } catch (IOException e) { + logger.debug("Unable to remove directory " + e); + } } } From 8212a36976dfcd829116621e5dc580575bd5c4e1 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 27 Jul 2020 13:11:46 -0500 Subject: [PATCH 110/655] Updated logic so the base directories are not removed if empty --- .../bioinformatics/irida/util/FileUtils.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index 74ce8037a11..fa9d8a403d9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -64,7 +64,7 @@ public static void removeTemporaryFile(Path fileToRemove){ String parentDirectoryPath = fileToRemove.getParent().toString(); Path dirToRemovePath = Paths.get(parentDirectoryPath); - if(fileToRemove.toFile().isFile()) { + if(Files.isRegularFile(fileToRemove)) { try { Files.delete(fileToRemove); } catch (IOException e) { @@ -72,16 +72,22 @@ public static void removeTemporaryFile(Path fileToRemove){ } } - // Directory is empty - if(dirToRemovePath.toFile().listFiles().length == 0) { - try { - org.apache.commons.io.FileUtils.deleteDirectory(dirToRemovePath.toFile()); - Path tDir = dirToRemovePath.toFile().toPath(); - if(tDir.toString().substring(tDir.toString().length() - 1).matches("\\d+")) { - removeTemporaryFile(tDir); + if(Files.isDirectory(dirToRemovePath)) { + boolean directoryIsNumeric = dirToRemovePath.toString() + .substring(dirToRemovePath.toString() + .length() - 1) + .matches("\\d+"); + // Directory is empty + if (directoryIsNumeric && dirToRemovePath.toFile() + .listFiles().length == 0) { + try { + org.apache.commons.io.FileUtils.deleteDirectory(dirToRemovePath.toFile()); + // Recusrsively call remoteTemporaryFile to remove all + // other empty numeric named parent directories + removeTemporaryFile(dirToRemovePath); + } catch (IOException e) { + logger.debug("Unable to remove directory " + e); } - } catch (IOException e) { - logger.debug("Unable to remove directory " + e); } } } From 0d236e1095ad7bcc7de3c3e62ac804f13f9dd2f5 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 27 Jul 2020 13:50:40 -0500 Subject: [PATCH 111/655] Removed unused import --- .../irida/pipeline/upload/galaxy/GalaxyHistoriesService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java index 5e7a825036a..7e12f8d2b05 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java @@ -30,7 +30,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.github.jmchilton.blend4j.galaxy.ToolsClient; From f73e8790f3fe80889325fd7a6bd85eeaace10461 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 31 Jul 2020 11:33:56 -0500 Subject: [PATCH 112/655] Updated file views that extend AbstractView to get files from local and cloud storage --- .../irida/model/sequenceFile/SequenceFile.java | 9 +++++++++ .../bioinformatics/irida/web/spring/view/CSVView.java | 9 +++++---- .../bioinformatics/irida/web/spring/view/FastaView.java | 8 +++++--- .../bioinformatics/irida/web/spring/view/FastqView.java | 6 +++--- .../irida/web/spring/view/GenbankView.java | 6 +++--- .../irida/web/spring/view/NewickFileView.java | 6 +++--- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index eb6f6497434..55c21c5210a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -314,4 +314,13 @@ public InputStream getFileInputStream() { return IridaFiles.getFileInputStream(file); } + /** + * Get file size in bytes + * + * @return if file exists or not + */ + public Long getFileSizeBytes() { + return IridaFiles.getFileSizeBytes(getFile()); + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java index 019e8757c83..fde93ba04c2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java @@ -1,13 +1,13 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.servlet.view.AbstractView; @@ -21,7 +21,8 @@ * Write out CSV files to the client. * */ -public class CSVView extends AbstractView { +public class +CSVView extends AbstractView { public static final String DEFAULT_CONTENT_TYPE = "text/csv"; private static final Logger logger = LoggerFactory.getLogger(CSVView.class); @@ -44,9 +45,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq logger.trace("Sending file to client [" + filename + "]"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); - response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(Files.size(fileContent))); + response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - Files.copy(fileContent, os); + IOUtils.copy(sfr.getFileInputStream(), os); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java index 02ad9139597..85f182992dc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java @@ -1,8 +1,11 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; import ca.corefacility.bioinformatics.irida.model.irida.IridaSequenceFile; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.controller.api.RESTGenericController; import com.google.common.net.HttpHeaders; + +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.servlet.view.AbstractView; @@ -10,7 +13,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -41,9 +43,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq logger.trace("Sending file to client [" + filename + "]"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); - response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(Files.size(fileContent))); + response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(IridaFiles.getFileSizeBytes(fileContent))); OutputStream os = response.getOutputStream(); - Files.copy(fileContent, os); + IOUtils.copy(IridaFiles.getFileInputStream(fileContent), os); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java index 2f8c23fe234..9e8c74106bb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java @@ -1,13 +1,13 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.servlet.view.AbstractView; @@ -43,9 +43,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq logger.trace("Sending file to client [" + filename + "]"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); - response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(Files.size(fileContent))); + response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - Files.copy(fileContent, os); + IOUtils.copy(sfr.getFileInputStream(), os); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java index b28cf993eac..22cfe9fe479 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java @@ -1,13 +1,13 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.servlet.view.AbstractView; @@ -44,9 +44,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq logger.trace("Sending file to client [" + filename + "]"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); - response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(Files.size(fileContent))); + response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - Files.copy(fileContent, os); + IOUtils.copy(sfr.getFileInputStream(), os); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java index 797c6447ae9..384d0efc091 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java @@ -1,13 +1,13 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.servlet.view.AbstractView; @@ -44,9 +44,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq logger.trace("Sending file to client [" + filename + "]"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); - response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(Files.size(fileContent))); + response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - Files.copy(fileContent, os); + IOUtils.copy(sfr.getFileInputStream(), os); os.flush(); os.close(); } From 0de23c848a94122c454d484ea95bdf5355a35257 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 31 Jul 2020 13:34:32 -0500 Subject: [PATCH 113/655] Updated to close inputstream on exit from method --- .../irida/processing/impl/ChecksumFileProcessor.java | 2 +- .../irida/processing/impl/GzipFileProcessor.java | 6 ++++-- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 4 +++- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 5 ++++- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 4 +++- .../irida/ria/utilities/FileUtilities.java | 12 +++++++++--- .../irida/ria/web/files/ReferenceFileController.java | 6 ++++-- .../irida/ria/web/files/SequenceFileController.java | 5 ++++- .../irida/ria/web/samples/SamplesController.java | 5 ++++- 9 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index 629b05c8e95..d1371a853bc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -58,8 +58,8 @@ public void process(SequencingObject sequencingObject) { String shaDigest = DigestUtils.sha256Hex(is); logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); - fileRepository.saveMetadata(file); + is.close(); } catch (IOException e) { throw new FileProcessorException("could not calculate checksum", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 8981f16e287..c2dfd1dd69a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.processing.impl; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -119,7 +120,8 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); - try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { + InputStream is = sequenceFile.getFileInputStream(); + try (GZIPInputStream zippedInputStream = new GZIPInputStream(is)) { logger.trace("Handling gzip compressed file."); Path targetDirectory = Files.createTempDirectory(null); @@ -128,7 +130,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc logger.debug("Writing uncompressed file to [" + target + "]"); Files.copy(zippedInputStream, target); - + is.close(); sequenceFile.setFile(target); sequenceFile = sequenceFileRepository.save(sequenceFile); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 93c9cd05249..e8271e9b07e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -171,8 +171,10 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream inputStream = getFileInputStream(file)) { byte[] bytes = new byte[2]; inputStream.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + boolean gzipped = ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + inputStream.close(); + return gzipped; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 07ef921d378..e0102bd5cb2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -62,6 +62,7 @@ public File getFile(Path file) { InputStream initialStream = blobClient.openInputStream(); File targetFile = new File(file.toAbsolutePath().toString()); FileUtils.copyInputStreamToFile(initialStream, targetFile); + initialStream.close(); fileToProcess = targetFile; } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); @@ -167,8 +168,10 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { byte[] bytes = new byte[2]; is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + boolean gzipped = ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + is.close(); + return gzipped; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 12825bc9ce4..2df3e651248 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -136,8 +136,10 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { byte[] bytes = new byte[2]; is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + boolean gzipped = ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + is.close(); + return gzipped; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java index bbc357e52a0..adc5c9d2b10 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java @@ -83,7 +83,9 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re outputStream.putNextEntry(new ZipEntry(zipEntryName.toString())); // 3) COPY all of thy bytes from the file to the output stream. - IOUtils.copy(file.getFileInputStream(),outputStream); + InputStream inputStream = file.getFileInputStream(); + IOUtils.copy(inputStream,outputStream); + inputStream.close(); // 4) Close the current entry in the archive in preparation for // the next entry. outputStream.closeEntry(); @@ -151,7 +153,9 @@ public static void createBatchAnalysisOutputFileZippedResponse(HttpServletRespon outputStream.putNextEntry(new ZipEntry(fileName + "/" + outputFilename)); // 3) COPY all of thy bytes from the file to the output stream. - IOUtils.copy(file.getFileInputStream(),outputStream); + InputStream inputStream = file.getFileInputStream(); + IOUtils.copy(inputStream,outputStream); + inputStream.close(); // 4) Close the current entry in the archive in preparation for // the next entry. @@ -187,7 +191,9 @@ public static void createSingleFileResponse(HttpServletResponse response, Analys response.setContentType(CONTENT_TYPE_TEXT); try (ServletOutputStream outputStream = response.getOutputStream()) { - IOUtils.copy(file.getFileInputStream(), outputStream); + InputStream inputStream = file.getFileInputStream(); + IOUtils.copy(inputStream, outputStream); + inputStream.close(); } catch (IOException e) { // this generally means that the user has cancelled the download // from their web browser; we can safely ignore this diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java index cec1cc2774c..714dd8653ce 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.web.files; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Date; @@ -72,10 +73,11 @@ public ReferenceFileController(ProjectService projectService, ReferenceFileServi public void downloadReferenceFile(@PathVariable Long fileId, HttpServletResponse response) throws IOException { ReferenceFile file = referenceFileService.read(fileId); - Path path = file.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getLabel() + "\""); - file.getFileInputStream().transferTo(response.getOutputStream()); + InputStream inputStream = file.getFileInputStream(); + inputStream.transferTo(response.getOutputStream()); response.flushBuffer(); + inputStream.close(); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java index a7b9b288886..63aae9cca00 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java @@ -3,6 +3,7 @@ import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.Date; @@ -129,8 +130,10 @@ public void downloadSequenceFile(@PathVariable Long sequencingObjectId, @PathVar SequenceFile sequenceFile = sequencingObject.getFileWithId(sequenceFileId); Path path = sequenceFile.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + sequenceFile.getLabel() + "\""); - sequenceFile.getFileInputStream().transferTo(response.getOutputStream()); + InputStream inputStream = sequenceFile.getFileInputStream(); + inputStream.transferTo(response.getOutputStream()); response.flushBuffer(); + inputStream.close(); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 915dec0e296..6f05298bcf9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.web.samples; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -356,8 +357,10 @@ public void downloadAssembly(@PathVariable Long sampleId, @PathVariable Long ass Path path = genomeAssembly.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + genomeAssembly.getLabel() + "\""); - genomeAssembly.getFileInputStream().transferTo(response.getOutputStream()); + InputStream inputStream = genomeAssembly.getFileInputStream(); + inputStream.transferTo(response.getOutputStream()); response.flushBuffer(); + inputStream.close(); } /** From 6deab9513a4b30d53fbc7f92a46659cf33fd1724 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 31 Jul 2020 13:40:13 -0500 Subject: [PATCH 114/655] Updated to close input stream after copying to outputstream --- .../bioinformatics/irida/web/spring/view/CSVView.java | 5 ++++- .../bioinformatics/irida/web/spring/view/FastaView.java | 6 +++++- .../bioinformatics/irida/web/spring/view/FastqView.java | 5 ++++- .../bioinformatics/irida/web/spring/view/GenbankView.java | 5 ++++- .../irida/web/spring/view/NewickFileView.java | 5 ++++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java index fde93ba04c2..2c4d001455f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; import java.util.Map; @@ -47,7 +48,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - IOUtils.copy(sfr.getFileInputStream(), os); + InputStream is = sfr.getFileInputStream(); + IOUtils.copy(is, os); + is.close(); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java index 85f182992dc..48e1a3de969 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java @@ -12,6 +12,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; + +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; import java.util.Map; @@ -45,7 +47,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(IridaFiles.getFileSizeBytes(fileContent))); OutputStream os = response.getOutputStream(); - IOUtils.copy(IridaFiles.getFileInputStream(fileContent), os); + InputStream is = IridaFiles.getFileInputStream(fileContent); + IOUtils.copy(is, os); + is.close(); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java index 9e8c74106bb..fdc4782481b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; import java.util.Map; @@ -45,7 +46,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - IOUtils.copy(sfr.getFileInputStream(), os); + InputStream is = sfr.getFileInputStream(); + IOUtils.copy(is, os); + is.close(); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java index 22cfe9fe479..87a5233bca1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; import java.util.Map; @@ -46,7 +47,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - IOUtils.copy(sfr.getFileInputStream(), os); + InputStream is = sfr.getFileInputStream(); + IOUtils.copy(is, os); + is.close(); os.flush(); os.close(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java index 384d0efc091..cf54f616c46 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; import java.util.Map; @@ -46,7 +47,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); OutputStream os = response.getOutputStream(); - IOUtils.copy(sfr.getFileInputStream(), os); + InputStream is = sfr.getFileInputStream(); + IOUtils.copy(is, os); + is.close(); os.flush(); os.close(); } From 23c0b6d48cdf222b4c5674c280f79ed361766082 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 4 Aug 2020 11:16:10 -0500 Subject: [PATCH 115/655] Removed @ResponseBody from rest controllers. Updated azure storage utility getTemporaryFile method. Removed duplicate imports --- .../IridaFileStorageAzureUtilityImpl.java | 35 +++++++++---------- .../RESTProjectAnalysisController.java | 2 -- .../RESTProjectSamplesController.java | 19 ---------- .../projects/RESTProjectUsersController.java | 3 -- .../api/projects/RESTProjectsController.java | 1 - 5 files changed, 16 insertions(+), 44 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 8d65c8a9573..d3e3436b112 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -7,7 +7,6 @@ import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Date; import java.util.List; import java.util.Optional; import java.util.zip.GZIPInputStream; @@ -36,7 +35,6 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient ; - private BlobClient blobClient; @Autowired public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerName){ @@ -53,21 +51,18 @@ public File getTemporaryFile(Path file) { File fileToProcess = null; // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { - // Create a file that will be unique in the /tmp/ folder. We append the current date/time - // to the file name - String tmpDir = "/tmp/" + new Date().toString().replaceAll("\\W", ""); - // Since the file system is virtual the full file path is the file name. - // We split it on "/" and get the last token which is the actual file name. - String [] blobNameTokens = blobClient.getBlobName().split("/"); - String fileName = blobNameTokens[blobNameTokens.length-1]; - String filePath = tmpDir + fileName; - blobClient.downloadToFile(filePath); - fileToProcess = new File(filePath); + InputStream initialStream = blobClient.openInputStream(); + File targetFile = new File(file.toAbsolutePath().toString()); + org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, targetFile); + initialStream.close(); + fileToProcess = targetFile; } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); + } catch (IOException e) { + logger.debug(e.getMessage()); } return fileToProcess; @@ -81,7 +76,7 @@ public String getFileSize(Path file) { String fileSize = "N/A"; try { // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties().getBlobSize(), true); } catch (BlobStorageException e) { logger.trace("Couldn't calculate size as the file was not found on azure [" + e + "]"); @@ -95,7 +90,7 @@ public String getFileSize(Path file) { @Override public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision) { // We set the blobClient "path" to which we want to upload our file to - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); try { logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); blobClient.uploadFromFile(source.toString(), false); @@ -118,7 +113,7 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { // Since the file system is virtual the full file path is the file name. // We split it on "/" and get the last token which is the actual file name. @@ -137,7 +132,7 @@ public String getFileName(Path file) { */ @Override public boolean fileExists(Path file) { - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); if(blobClient.exists()) { return true; } @@ -149,12 +144,14 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { + BlobClient blobClient; try { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + return blobClient.openInputStream(); } catch (BlobStorageException e) { logger.trace("Couldn't read file from azure [" + e + "]"); } - return blobClient.openInputStream(); + return null; } /** @@ -186,8 +183,8 @@ private String getAzureFileAbsolutePath(Path file) { } return absolutePath; } - /** + * {@inheritDoc} */ @Override diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java index 1a1bc52e2ba..cbaeb83202f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java @@ -66,7 +66,6 @@ public RESTProjectAnalysisController(ProjectService projectService, * {@link Project}. */ @RequestMapping(value = "/api/projects/{projectId}/analyses", method = RequestMethod.GET) - @ResponseBody public ModelMap getProjectAnalyses(@PathVariable Long projectId) { logger.debug("Loading analyses for project [" + projectId + "]"); @@ -109,7 +108,6 @@ public ModelMap getProjectAnalyses(@PathVariable Long projectId) { * found in IRIDA. */ @RequestMapping(value = "/api/projects/{projectId}/analyses/{type}", method = RequestMethod.GET) - @ResponseBody public ModelMap getProjectAnalysesByType(@PathVariable Long projectId, @PathVariable String type) throws IridaWorkflowNotFoundException { logger.debug("Loading analyses for project [" + projectId + "] by type [" + type + "]"); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java index f5d5255e515..ab7ab0beac7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java @@ -36,17 +36,6 @@ import ca.corefacility.bioinformatics.irida.web.controller.api.samples.RESTSampleSequenceFilesController; import com.google.common.net.HttpHeaders; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.hateoas.Link; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Controller; -import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.servlet.ModelAndView; -import org.springframework.web.servlet.view.RedirectView; -import com.google.common.net.HttpHeaders; - import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; @@ -100,7 +89,6 @@ public RESTProjectSamplesController(ProjectService projectService, SampleService * project. */ @RequestMapping(value = "/api/projects/{projectId}/samples", method = RequestMethod.POST, consumes = "application/idcollection+json") - @ResponseBody public ModelMap copySampleToProject(final @PathVariable Long projectId, final @RequestBody List sampleIds, HttpServletResponse response) { @@ -155,7 +143,6 @@ public ModelMap copySampleToProject(final @PathVariable Long projectId, final @R * location information. */ @RequestMapping(value = "/api/projects/{projectId}/samples", method = RequestMethod.POST, consumes = "!application/idcollection+json") - @ResponseBody public ModelMap addSampleToProject(@PathVariable Long projectId, @RequestBody Sample sample, HttpServletResponse response) { ModelMap model = new ModelMap(); @@ -193,7 +180,6 @@ public ModelMap addSampleToProject(@PathVariable Long projectId, @RequestBody Sa * @return the list of {@link Sample}s associated with this {@link Project}. */ @RequestMapping(value = "/api/projects/{projectId}/samples", method = RequestMethod.GET) - @ResponseBody public ModelMap getProjectSamples(@PathVariable Long projectId) { ModelMap modelMap = new ModelMap(); @@ -223,7 +209,6 @@ public ModelMap getProjectSamples(@PathVariable Long projectId) { * @return The found sample */ @RequestMapping(value = "/api/projects/{projectId}/samples/bySequencerId/{seqeuncerId}", method = RequestMethod.GET) - @ResponseBody public ModelAndView getProjectSampleBySequencerId(@PathVariable Long projectId, @PathVariable String seqeuncerId) { Project p = projectService.read(projectId); @@ -251,7 +236,6 @@ public ModelAndView getProjectSampleBySequencerId(@PathVariable Long projectId, * @return a representation of the specific sample. */ @RequestMapping(value = "/api/projects/{projectId}/samples/{sampleId}", method = RequestMethod.GET) - @ResponseBody public ModelMap getProjectSample(@PathVariable Long projectId, @PathVariable Long sampleId) { // read project/sample to verify sample exists in project Project project = projectService.read(projectId); @@ -278,7 +262,6 @@ public ModelMap getProjectSample(@PathVariable Long projectId, @PathVariable Lon * @return representation of the sample */ @RequestMapping(value = "/api/samples/{sampleId}", method = RequestMethod.GET) - @ResponseBody public ModelMap getSample(@PathVariable Long sampleId) { ModelMap modelMap = new ModelMap(); Sample s = sampleService.read(sampleId); @@ -336,7 +319,6 @@ private void addLinksForSample(final Optional p, final Sample s) { * and collection of {@link Sample}. */ @RequestMapping(value = "/api/projects/{projectId}/samples/{sampleId}", method = RequestMethod.DELETE) - @ResponseBody public ModelMap removeSampleFromProject(@PathVariable Long projectId, @PathVariable Long sampleId) { ModelMap modelMap = new ModelMap(); @@ -373,7 +355,6 @@ public ModelMap removeSampleFromProject(@PathVariable Long projectId, @PathVaria */ @RequestMapping(value = "/api/samples/{sampleId}", method = RequestMethod.PATCH, consumes = { MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE }) - @ResponseBody public ModelMap updateSample(@PathVariable Long sampleId, @RequestBody Map updatedFields) { ModelMap modelMap = new ModelMap(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java index c70aa9efaad..0f02958d928 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java @@ -76,7 +76,6 @@ public RESTProjectUsersController(UserService userService, ProjectService projec * thrown in this method, but needs to be listed. */ @RequestMapping(value = "/api/projects/{projectId}/users", method = RequestMethod.GET) - @ResponseBody public ModelMap getUsersForProject(@PathVariable Long projectId) throws ProjectWithoutOwnerException { ResourceCollection resources = new ResourceCollection<>(); @@ -119,7 +118,6 @@ public ModelMap getUsersForProject(@PathVariable Long projectId) throws ProjectW * {@code linkTo} and {@code methodOn}. */ @RequestMapping(value = "/api/projects/{projectId}/users", method = RequestMethod.POST) - @ResponseBody public ModelMap addUserToProject(@PathVariable Long projectId, @RequestBody Map representation, HttpServletResponse response) throws ProjectWithoutOwnerException { // first, get the project @@ -176,7 +174,6 @@ public ModelMap addUserToProject(@PathVariable Long projectId, @RequestBody Map< * @throws ProjectWithoutOwnerException if removing this user will leave the project without an owner */ @RequestMapping(value = "/api/projects/{projectId}/users/{userId}", method = RequestMethod.DELETE) - @ResponseBody public ModelMap removeUserFromProject(@PathVariable Long projectId, @PathVariable String userId) throws ProjectWithoutOwnerException { // Read the project and user from the database diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java index da0f8e1046c..d805862cd41 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java @@ -24,7 +24,6 @@ * */ @Controller -@ResponseBody @RequestMapping(value = "/api/projects") public class RESTProjectsController extends RESTGenericController { From 2fbdcba5960698104c64d60d0dc1e65ac704d670 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 4 Aug 2020 11:28:10 -0500 Subject: [PATCH 116/655] Removed unused imports --- .../controller/api/projects/RESTProjectAnalysisController.java | 1 - .../web/controller/api/projects/RESTProjectsController.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java index cbaeb83202f..48b731c8531 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectAnalysisController.java @@ -13,7 +13,6 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException; import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java index d805862cd41..669de3f6329 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectsController.java @@ -12,7 +12,6 @@ import org.springframework.hateoas.Link; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.ResponseBody; import ca.corefacility.bioinformatics.irida.exceptions.ProjectWithoutOwnerException; import ca.corefacility.bioinformatics.irida.model.project.Project; From fdd25e1d2dc4f0342951f50e259cf2ec0657dfb2 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 4 Aug 2020 11:46:47 -0500 Subject: [PATCH 117/655] Updated variable name --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 2 +- .../repositories/filesystem/IridaFileStorageUtility.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index d3e3436b112..7eaabf3f3a5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -215,7 +215,7 @@ public String getFileExtension(List sequencingObject for (SequenceFile file : object.getFiles()) { String fileName = getFileName(file.getFile()); - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + Optional currentExtensionOpt = VALID_CONCATENATION_EXTENSIONS.stream() .filter(e -> fileName.endsWith(e)) .findFirst(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 1c7133cd241..6d26f54c5fc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -16,8 +16,8 @@ */ public interface IridaFileStorageUtility { - //Valid extensions to try to concatenate with this tool - public static final List VALID_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); + //Valid file extensions for sample file concatenation + public static final List VALID_CONCATENATION_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** * Get a temporarry file from storage * From 51d1b298f6f6d0e5746810c1742022ea2c4a1d9e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 4 Aug 2020 12:54:09 -0500 Subject: [PATCH 118/655] Updated variable name --- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 4316a094e79..578f6bd464d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -173,7 +173,7 @@ public String getFileExtension(List sequencingObject .toFile() .getName(); - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + Optional currentExtensionOpt = VALID_CONCATENATION_EXTENSIONS.stream() .filter(e -> fileName.endsWith(e)) .findFirst(); From 234e06d6ef3aa99439c6a7a1eb24af82b935c95b Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 11 Aug 2020 17:14:06 -0500 Subject: [PATCH 119/655] Changed FileSystemResource to InputStreamSource to enable reading analysis output files from an object store --- .../api/RESTAnalysisSubmissionController.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index bc976c79d5b..1cfc79b6ca9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -15,6 +15,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; import ca.corefacility.bioinformatics.irida.model.workflow.description.IridaWorkflowDescription; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -24,6 +25,8 @@ import com.google.common.collect.ImmutableMap; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.InputStreamSource; import org.springframework.hateoas.Link; import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; @@ -49,6 +52,7 @@ public class RESTAnalysisSubmissionController extends RESTGenericController Date: Tue, 11 Aug 2020 17:16:13 -0500 Subject: [PATCH 120/655] Removed iridaFileStorageUtility which was wired in as it is not necessary --- .../controller/api/RESTAnalysisSubmissionController.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index 1cfc79b6ca9..cf40329b99c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -15,7 +15,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; import ca.corefacility.bioinformatics.irida.model.workflow.description.IridaWorkflowDescription; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -24,7 +23,6 @@ import ca.corefacility.bioinformatics.irida.web.controller.api.samples.RESTSampleSequenceFilesController; import com.google.common.collect.ImmutableMap; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamSource; import org.springframework.hateoas.Link; @@ -52,7 +50,6 @@ public class RESTAnalysisSubmissionController extends RESTGenericController Date: Wed, 12 Aug 2020 10:04:46 -0500 Subject: [PATCH 121/655] Removed dependency not required. Updated comments --- pom.xml | 6 +----- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index a6a69032fa4..c0e553e49e9 100644 --- a/pom.xml +++ b/pom.xml @@ -344,11 +344,7 @@ jackson-databind ${jackson.version} - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - + com.fasterxml.jackson.dataformat jackson-dataformat-yaml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 7eaabf3f3a5..5f4bd42d893 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -50,7 +50,7 @@ public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerNa public File getTemporaryFile(Path file) { File fileToProcess = null; - // We set the blobClient "path" to which we want to upload our file to + // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { @@ -75,7 +75,7 @@ public File getTemporaryFile(Path file) { public String getFileSize(Path file) { String fileSize = "N/A"; try { - // We set the blobClient "path" to which we want to upload our file to + // We set the blobClient "path" to which we want to get a file size for BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties().getBlobSize(), true); } catch (BlobStorageException e) { From 1d77db91ce07dee1b577890082d13ee5360a0717 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 12 Aug 2020 10:08:29 -0500 Subject: [PATCH 122/655] Removed @ResponseBody annotation not required --- .../irida/web/controller/api/RESTRootController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java index 739883f585d..ecfa5b822bb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTRootController.java @@ -84,7 +84,6 @@ public void initLinks() { * @return a response to the client. */ @RequestMapping(method = RequestMethod.GET, value = "/api") - @ResponseBody public ModelMap getLinks(final HttpServletRequest request) { logger.debug("Discovering application"); RootResource resource = new RootResource(); From 4edd211625ed2c7284e7faa9c8bc68524c5e874f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 12 Aug 2020 11:49:36 -0500 Subject: [PATCH 123/655] Updated variable name --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 2 +- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 2 +- .../repositories/filesystem/IridaFileStorageUtility.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 2da1b7c3e2d..ea7affbdd92 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -220,7 +220,7 @@ public String getFileExtension(List sequencingObject for (SequenceFile file : object.getFiles()) { String fileName = getFileName(file.getFile()); - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + Optional currentExtensionOpt = VALID_CONCATENATION_EXTENSIONS.stream() .filter(e -> fileName.endsWith(e)) .findFirst(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 4316a094e79..578f6bd464d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -173,7 +173,7 @@ public String getFileExtension(List sequencingObject .toFile() .getName(); - Optional currentExtensionOpt = VALID_EXTENSIONS.stream() + Optional currentExtensionOpt = VALID_CONCATENATION_EXTENSIONS.stream() .filter(e -> fileName.endsWith(e)) .findFirst(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 1c7133cd241..6d26f54c5fc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -16,8 +16,8 @@ */ public interface IridaFileStorageUtility { - //Valid extensions to try to concatenate with this tool - public static final List VALID_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); + //Valid file extensions for sample file concatenation + public static final List VALID_CONCATENATION_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** * Get a temporarry file from storage * From bdee3fd5ecbe7c0e5096d3f5889a51af59eac4f2 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 18 Aug 2020 16:22:16 -0500 Subject: [PATCH 124/655] Updated to use temporary files which are cleaned up after they have been used --- .../processing/impl/FastqcFileProcessor.java | 8 +++++- .../IridaFileStorageAzureUtilityImpl.java | 26 ++++++++++++++++--- .../IridaFileStorageLocalUtilityImpl.java | 8 ++++++ .../filesystem/IridaFileStorageUtility.java | 7 +++++ .../bioinformatics/irida/util/IridaFiles.java | 6 +++++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index b8c20202d54..a3cb636bf89 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -12,6 +12,7 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +33,7 @@ import javax.imageio.ImageIO; import java.awt.*; import java.awt.image.BufferedImage; +import java.io.File; import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; @@ -96,8 +98,9 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx .description(messageSource.getMessage("fastqc.file.processor.analysis.description", new Object[] {FastQCApplication.VERSION}, LocaleContextHolder.getLocale())); try { + File fastQCSequenceFileToProcess = iridaFileStorageUtility.getTemporaryFile(fileToProcess); uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - iridaFileStorageUtility.getTemporaryFile(fileToProcess)); + fastQCSequenceFileToProcess); BasicStats basicStats = new BasicStats(); PerBaseQualityScores pbqs = new PerBaseQualityScores(); PerSequenceQualityScores psqs = new PerSequenceQualityScores(); @@ -130,6 +133,9 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx sequenceFile.setFastQCAnalysis(analysis.build()); sequenceFileRepository.saveMetadata(sequenceFile); + + IridaFiles.cleanupLocalFiles(fastQCSequenceFileToProcess.toPath()); + IridaFiles.cleanupLocalFiles(outputDirectory); } catch (Exception e) { logger.error("FastQC failed to process the sequence file: " + e.getMessage()); throw new FileProcessorException("FastQC failed to parse the sequence file.", e); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 5f4bd42d893..54b64ddc19a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.List; @@ -54,11 +55,12 @@ public File getTemporaryFile(Path file) { BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { + Path tempDirectory = Files.createTempDirectory(null); + Path tempFile = tempDirectory.resolve(file.getFileName().toString()); InputStream initialStream = blobClient.openInputStream(); - File targetFile = new File(file.toAbsolutePath().toString()); - org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, targetFile); + org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); initialStream.close(); - fileToProcess = targetFile; + fileToProcess = tempFile.toFile(); } catch (BlobStorageException e) { logger.trace("Couldn't find file on azure [" + e + "]"); } catch (IOException e) { @@ -68,6 +70,23 @@ public File getTemporaryFile(Path file) { return fileToProcess; } + /** + * {@inheritDoc} + */ + @Override + public void cleanupLocalFiles(Path path) { + logger.trace("Cleaning up temporary file downloaded from azure [" + path.toString() + "]"); + try { + if(Files.isRegularFile(path)) { + Files.delete(path); + } else { + org.apache.commons.io.FileUtils.deleteDirectory(path.toFile()); + } + } catch (IOException e) { + logger.error("Unable to delete local file [" + path.toString() + "]"); + } + } + /** * {@inheritDoc} */ @@ -98,6 +117,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque } catch (BlobStorageException e) { logger.trace("Unable to upload file to azure [" + e + "]"); } + cleanupLocalFiles(source); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 578f6bd464d..1600235bcc0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -44,6 +44,14 @@ public File getTemporaryFile(Path file) { return fileToProcess; } + /** + * {@inheritDoc} + */ + @Override + public void cleanupLocalFiles(Path path) { + logger.trace("File resides on local filesystem. Not cleaning up file [" + path.toString() + "]"); + } + /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 6d26f54c5fc..272f335ea57 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -26,6 +26,13 @@ public interface IridaFileStorageUtility { */ public File getTemporaryFile(Path file); + /** + * Delete temporary file. + * + * @param file The {@link Path} to the file + */ + public void cleanupLocalFiles(Path file); + /** * Get file size * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 21efbc244ac..d676014fd71 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -65,4 +65,10 @@ public static String getFileExtension(List files) th return iridaFileStorageUtility.getFileExtension(files); } + /** + * Cleans up temporary downloaded files + * @param file The path to the file + */ + public static void cleanupLocalFiles(Path file) { iridaFileStorageUtility.cleanupLocalFiles(file); } + } From 4b16c81a35cbfe72080cf3818046d989db28fcad Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 18 Aug 2020 16:36:51 -0500 Subject: [PATCH 125/655] Updated to throw a storageexception if file is not found on azure rather than just logging a message --- .../IridaFileStorageAzureUtilityImpl.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 54b64ddc19a..d7b827c3369 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -17,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.util.FileUtils; @@ -55,6 +56,7 @@ public File getTemporaryFile(Path file) { BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { + logger.trace("Getting file from azure [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory(null); Path tempFile = tempDirectory.resolve(file.getFileName().toString()); InputStream initialStream = blobClient.openInputStream(); @@ -62,9 +64,10 @@ public File getTemporaryFile(Path file) { initialStream.close(); fileToProcess = tempFile.toFile(); } catch (BlobStorageException e) { - logger.trace("Couldn't find file on azure [" + e + "]"); + logger.error("Couldn't find file on azure [" + e + "]"); + throw new StorageException("Unable to locate file on azure", e); } catch (IOException e) { - logger.debug(e.getMessage()); + logger.error(e.getMessage()); } return fileToProcess; @@ -98,7 +101,8 @@ public String getFileSize(Path file) { BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties().getBlobSize(), true); } catch (BlobStorageException e) { - logger.trace("Couldn't calculate size as the file was not found on azure [" + e + "]"); + logger.error("Couldn't calculate size as the file was not found on azure [" + e + "]"); + throw new StorageException("Unable to locate file on azure", e); } return fileSize; } @@ -115,7 +119,8 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque blobClient.uploadFromFile(source.toString(), false); logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); } catch (BlobStorageException e) { - logger.trace("Unable to upload file to azure [" + e + "]"); + logger.error("Unable to upload file to azure [" + e + "]"); + throw new StorageException("Unable to upload file to azure", e); } cleanupLocalFiles(source); } @@ -141,7 +146,8 @@ public String getFileName(Path file) { .split("/"); fileName = blobNameTokens[blobNameTokens.length - 1]; } catch (BlobStorageException e) { - logger.trace("Couldn't find file on azure [" + e + "]"); + logger.error("Couldn't find file on azure [" + e + "]"); + throw new StorageException("Unable to locate file on azure", e); } return fileName; @@ -166,12 +172,13 @@ public boolean fileExists(Path file) { public InputStream getFileInputStream(Path file) { BlobClient blobClient; try { + logger.trace("Opening input stream to file on azure [" + file.toString() + "]"); blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); return blobClient.openInputStream(); } catch (BlobStorageException e) { - logger.trace("Couldn't read file from azure [" + e + "]"); + logger.error("Couldn't read file from azure [" + e + "]"); + throw new StorageException("Unable to locate file on azure", e); } - return null; } /** From d1684d340301b0ee6445ac72a903ac6019342a4c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 20 Aug 2020 15:12:16 -0500 Subject: [PATCH 126/655] Updated FastQCFileProcessor to clean up temp files in a finally block. Updated cleanupLocalFiles method to removed temp directories --- .../processing/impl/FastqcFileProcessor.java | 83 +++++++++++-------- .../IridaFileStorageAzureUtilityImpl.java | 20 ++++- 2 files changed, 65 insertions(+), 38 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index a3cb636bf89..723d75f0420 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.processing.impl; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -97,48 +98,62 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx .executionManagerAnalysisId(EXECUTION_MANAGER_ANALYSIS_ID) .description(messageSource.getMessage("fastqc.file.processor.analysis.description", new Object[] {FastQCApplication.VERSION}, LocaleContextHolder.getLocale())); - try { - File fastQCSequenceFileToProcess = iridaFileStorageUtility.getTemporaryFile(fileToProcess); - uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( - fastQCSequenceFileToProcess); - BasicStats basicStats = new BasicStats(); - PerBaseQualityScores pbqs = new PerBaseQualityScores(); - PerSequenceQualityScores psqs = new PerSequenceQualityScores(); - OverRepresentedSeqs overRep = new OverRepresentedSeqs(); - QCModule[] moduleList = new QCModule[] { basicStats, pbqs, psqs, overRep }; - - logger.debug("Launching FastQC analysis modules on all sequences."); - while (fastQCSequenceFile.hasNext()) { - Sequence sequence = fastQCSequenceFile.next(); - for (QCModule module : moduleList) { - module.processSequence(sequence); - } - } - logger.debug("Finished FastQC analysis modules."); + File fastQCSequenceFileToProcess = iridaFileStorageUtility.getTemporaryFile(fileToProcess); + Path perBaseQualityScoresOutputDirectory = null; + Path perSequenceQualityScoresOutputDirectory = null; + Path duplicationLevelOutputDirectory = null; - Path outputDirectory = Files.createTempDirectory("analysis-output"); + try { + perBaseQualityScoresOutputDirectory = Files.createTempDirectory("analysis-output"); + perSequenceQualityScoresOutputDirectory = Files.createTempDirectory("analysis-output"); + duplicationLevelOutputDirectory = Files.createTempDirectory("analysis-output"); + + try { + uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( + fastQCSequenceFileToProcess); + BasicStats basicStats = new BasicStats(); + PerBaseQualityScores pbqs = new PerBaseQualityScores(); + PerSequenceQualityScores psqs = new PerSequenceQualityScores(); + OverRepresentedSeqs overRep = new OverRepresentedSeqs(); + QCModule[] moduleList = new QCModule[] { basicStats, pbqs, psqs, overRep }; + + logger.debug("Launching FastQC analysis modules on all sequences."); + while (fastQCSequenceFile.hasNext()) { + Sequence sequence = fastQCSequenceFile.next(); + for (QCModule module : moduleList) { + module.processSequence(sequence); + } + } - handleBasicStats(basicStats, analysis); - handlePerBaseQualityScores(pbqs, analysis, outputDirectory); - handlePerSequenceQualityScores(psqs, analysis, outputDirectory); - handleDuplicationLevel(overRep.duplicationLevelModule(), analysis, outputDirectory); - Set overrepresentedSequences = handleOverRepresentedSequences(overRep); + logger.debug("Finished FastQC analysis modules."); - logger.trace("Saving FastQC analysis."); - analysis.overrepresentedSequences(overrepresentedSequences); + handleBasicStats(basicStats, analysis); + handlePerBaseQualityScores(pbqs, analysis, perBaseQualityScoresOutputDirectory); + handlePerSequenceQualityScores(psqs, analysis, perSequenceQualityScoresOutputDirectory); + handleDuplicationLevel(overRep.duplicationLevelModule(), analysis, duplicationLevelOutputDirectory); + Set overrepresentedSequences = handleOverRepresentedSequences(overRep); - AnalysisFastQC analysisFastQC = analysis.build(); + logger.trace("Saving FastQC analysis."); + analysis.overrepresentedSequences(overrepresentedSequences); - sequenceFile.setFastQCAnalysis(analysis.build()); + AnalysisFastQC analysisFastQC = analysis.build(); - sequenceFileRepository.saveMetadata(sequenceFile); + sequenceFile.setFastQCAnalysis(analysis.build()); - IridaFiles.cleanupLocalFiles(fastQCSequenceFileToProcess.toPath()); - IridaFiles.cleanupLocalFiles(outputDirectory); - } catch (Exception e) { - logger.error("FastQC failed to process the sequence file: " + e.getMessage()); - throw new FileProcessorException("FastQC failed to parse the sequence file.", e); + sequenceFileRepository.saveMetadata(sequenceFile); + } catch (Exception e) { + logger.error("FastQC failed to process the sequence file: " + e.getMessage()); + throw new FileProcessorException("FastQC failed to parse the sequence file.", e); + } + } catch (IOException e) { + logger.error("Unable to create temporary directory ", e); + throw new StorageException("Unable to create temporary directory", e); + } finally { + IridaFiles.cleanupLocalFiles(fastQCSequenceFileToProcess.toPath()); + IridaFiles.cleanupLocalFiles(perBaseQualityScoresOutputDirectory); + IridaFiles.cleanupLocalFiles(perSequenceQualityScoresOutputDirectory); + IridaFiles.cleanupLocalFiles(duplicationLevelOutputDirectory); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index d7b827c3369..83984659eba 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -79,11 +79,22 @@ public File getTemporaryFile(Path file) { @Override public void cleanupLocalFiles(Path path) { logger.trace("Cleaning up temporary file downloaded from azure [" + path.toString() + "]"); + + Path origPath = path; + if(Files.isRegularFile(path)) { + origPath = path; + path = path.getParent(); + } + try { - if(Files.isRegularFile(path)) { - Files.delete(path); - } else { - org.apache.commons.io.FileUtils.deleteDirectory(path.toFile()); + if(Files.isDirectory(path)) { + // Only delete the directory if it is not the root directory + if(path.getRoot() != path) { + org.apache.commons.io.FileUtils.deleteDirectory(path.toFile()); + } else { + // We are already in the root directory so just delete the file + Files.delete(origPath); + } } } catch (IOException e) { logger.error("Unable to delete local file [" + path.toString() + "]"); @@ -122,6 +133,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque logger.error("Unable to upload file to azure [" + e + "]"); throw new StorageException("Unable to upload file to azure", e); } + // Removes the parent directory and file cleanupLocalFiles(source); } From 53de67fc0a451b065673734e8b36a977b918e33f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 20 Aug 2020 15:34:04 -0500 Subject: [PATCH 127/655] Removed unused import --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index f3e8a1937a7..fd920f0661a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.List; From 5988e06b794923529c3dd6889ff54b983970516a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 25 Aug 2020 11:32:55 -0500 Subject: [PATCH 128/655] Changed from throwing ioexception to storageexception in azure file storage utility class --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 83984659eba..8c0b6b64edf 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -68,6 +68,7 @@ public File getTemporaryFile(Path file) { throw new StorageException("Unable to locate file on azure", e); } catch (IOException e) { logger.error(e.getMessage()); + throw new StorageException(e.getMessage()); } return fileToProcess; @@ -98,6 +99,7 @@ public void cleanupLocalFiles(Path path) { } } catch (IOException e) { logger.error("Unable to delete local file [" + path.toString() + "]"); + throw new StorageException(e.getMessage()); } } @@ -235,11 +237,11 @@ public void appendToFile(Path target, SequenceFile file) throws IOException{ p += in.transferTo(p, l - p, out); } } catch (IOException e) { - throw new IOException("Could not open input file for reading", e); + throw new StorageException("Could not open input file for reading", e); } } catch (IOException e) { - throw new IOException("Could not open target file for writing", e); + throw new StorageException("Could not open target file for writing", e); } } From 46fe13d1c605f4f904ff4b5ec6f433dc322d8737 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 31 Aug 2020 17:29:11 -0500 Subject: [PATCH 129/655] Updated getTemporaryFile method to return an IridaTemporaryFile object which has the path to the file and temporary directory. Updating directory removal --- .../processing/impl/FastqcFileProcessor.java | 24 +++----- .../processing/impl/GzipFileProcessor.java | 61 +++++++++++-------- .../IridaFileStorageAzureUtilityImpl.java | 50 ++++++++------- .../IridaFileStorageLocalUtilityImpl.java | 14 +++-- .../filesystem/IridaFileStorageUtility.java | 10 +-- .../irida/ria/web/dto/IridaTemporaryFile.java | 33 ++++++++++ .../web/samples/SamplesAjaxController.java | 27 +++++--- .../bioinformatics/irida/util/IridaFiles.java | 12 +++- .../samples/SamplesAjaxControllerTest.java | 6 ++ 9 files changed, 150 insertions(+), 87 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 723d75f0420..a09309f648d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -13,6 +13,7 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import org.slf4j.Logger; @@ -99,15 +100,12 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx .description(messageSource.getMessage("fastqc.file.processor.analysis.description", new Object[] {FastQCApplication.VERSION}, LocaleContextHolder.getLocale())); - File fastQCSequenceFileToProcess = iridaFileStorageUtility.getTemporaryFile(fileToProcess); - Path perBaseQualityScoresOutputDirectory = null; - Path perSequenceQualityScoresOutputDirectory = null; - Path duplicationLevelOutputDirectory = null; + IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(fileToProcess); + File fastQCSequenceFileToProcess = iridaTemporaryFile.getFile().toFile(); + Path outputDirectory = null; try { - perBaseQualityScoresOutputDirectory = Files.createTempDirectory("analysis-output"); - perSequenceQualityScoresOutputDirectory = Files.createTempDirectory("analysis-output"); - duplicationLevelOutputDirectory = Files.createTempDirectory("analysis-output"); + outputDirectory = Files.createTempDirectory("analysis-output"); try { uk.ac.babraham.FastQC.Sequence.SequenceFile fastQCSequenceFile = SequenceFactory.getSequenceFile( @@ -129,9 +127,9 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx logger.debug("Finished FastQC analysis modules."); handleBasicStats(basicStats, analysis); - handlePerBaseQualityScores(pbqs, analysis, perBaseQualityScoresOutputDirectory); - handlePerSequenceQualityScores(psqs, analysis, perSequenceQualityScoresOutputDirectory); - handleDuplicationLevel(overRep.duplicationLevelModule(), analysis, duplicationLevelOutputDirectory); + handlePerBaseQualityScores(pbqs, analysis, outputDirectory); + handlePerSequenceQualityScores(psqs, analysis, outputDirectory); + handleDuplicationLevel(overRep.duplicationLevelModule(), analysis, outputDirectory); Set overrepresentedSequences = handleOverRepresentedSequences(overRep); logger.trace("Saving FastQC analysis."); @@ -150,10 +148,8 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx logger.error("Unable to create temporary directory ", e); throw new StorageException("Unable to create temporary directory", e); } finally { - IridaFiles.cleanupLocalFiles(fastQCSequenceFileToProcess.toPath()); - IridaFiles.cleanupLocalFiles(perBaseQualityScoresOutputDirectory); - IridaFiles.cleanupLocalFiles(perSequenceQualityScoresOutputDirectory); - IridaFiles.cleanupLocalFiles(duplicationLevelOutputDirectory); + IridaFiles.cleanupLocalFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalFiles(null, outputDirectory); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 8981f16e287..3a44d8bb986 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -6,6 +6,7 @@ import java.nio.file.Paths; import java.util.zip.GZIPInputStream; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.Fast5Object; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,7 +20,7 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; - +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Handle gzip-ed files (if necessary). This class partially assumes that gzip @@ -114,36 +115,44 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try { logger.trace("About to try handling a gzip file."); - if (sequenceFile.isGzipped()) { file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); - - try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { - logger.trace("Handling gzip compressed file."); - - Path targetDirectory = Files.createTempDirectory(null); - Path target = targetDirectory.resolve(nameWithoutExtension); - logger.debug("Target directory is [" + targetDirectory + "]"); - logger.debug("Writing uncompressed file to [" + target + "]"); - - Files.copy(zippedInputStream, target); - - sequenceFile.setFile(target); - sequenceFile = sequenceFileRepository.save(sequenceFile); - - if (removeCompressedFile) { - logger.debug( - "Removing original compressed files [file.processing.decompress.remove.compressed.file=true]"); - try { - Files.delete(file); - } catch (final Exception e) { - logger.error("Failed to remove the original compressed file.", e); - // throw the exception again to be caught by the - // outer try/catch block: - throw e; + Path targetDirectory = null; + + try { + targetDirectory = Files.createTempDirectory(null); + + try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { + logger.trace("Handling gzip compressed file."); + + Path target = targetDirectory.resolve(nameWithoutExtension); + logger.debug("Target directory is [" + targetDirectory + "]"); + logger.debug("Writing uncompressed file to [" + target + "]"); + + Files.copy(zippedInputStream, target); + + sequenceFile.setFile(target); + sequenceFile = sequenceFileRepository.save(sequenceFile); + + if (removeCompressedFile) { + logger.debug( + "Removing original compressed files [file.processing.decompress.remove.compressed.file=true]"); + try { + Files.delete(file); + } catch (final Exception e) { + logger.error("Failed to remove the original compressed file.", e); + // throw the exception again to be caught by the + // outer try/catch block: + throw e; + } } } + } catch(IOException e) { + logger.error("Unable to create temporary directory", e); + throw new StorageException("Unable to create temporary directory"); + } finally { + IridaFiles.cleanupLocalFiles(null, targetDirectory); } } } catch (Exception e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 8c0b6b64edf..2f044ba34af 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -20,7 +20,9 @@ import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.util.FileUtils; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -49,9 +51,7 @@ public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerNa * {@inheritDoc} */ @Override - public File getTemporaryFile(Path file) { - File fileToProcess = null; - + public IridaTemporaryFile getTemporaryFile(Path file) { // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); @@ -62,7 +62,7 @@ public File getTemporaryFile(Path file) { InputStream initialStream = blobClient.openInputStream(); org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); initialStream.close(); - fileToProcess = tempFile.toFile(); + return new IridaTemporaryFile(tempFile, tempDirectory); } catch (BlobStorageException e) { logger.error("Couldn't find file on azure [" + e + "]"); throw new StorageException("Unable to locate file on azure", e); @@ -70,35 +70,30 @@ public File getTemporaryFile(Path file) { logger.error(e.getMessage()); throw new StorageException(e.getMessage()); } - - return fileToProcess; } /** * {@inheritDoc} */ @Override - public void cleanupLocalFiles(Path path) { - logger.trace("Cleaning up temporary file downloaded from azure [" + path.toString() + "]"); - - Path origPath = path; - if(Files.isRegularFile(path)) { - origPath = path; - path = path.getParent(); + public void cleanupLocalFiles(Path filePath, Path directoryPath) { + try { + if(filePath != null && Files.isRegularFile(filePath)) { + logger.trace("Cleaning up temporary file downloaded from azure [" + filePath.toString() + "]"); + Files.delete(filePath); + } + } catch (IOException e) { + logger.error("Unable to delete local file [" + filePath.toString() + "]"); + throw new StorageException(e.getMessage()); } try { - if(Files.isDirectory(path)) { - // Only delete the directory if it is not the root directory - if(path.getRoot() != path) { - org.apache.commons.io.FileUtils.deleteDirectory(path.toFile()); - } else { - // We are already in the root directory so just delete the file - Files.delete(origPath); - } + if(directoryPath != null && Files.isDirectory(directoryPath)) { + logger.trace("Cleaning up temporary directory [" + directoryPath.toString() + "]"); + org.apache.commons.io.FileUtils.deleteDirectory(directoryPath.toFile()); } } catch (IOException e) { - logger.error("Unable to delete local file [" + path.toString() + "]"); + logger.error("Unable to delete local directory [" + directoryPath.toString() + "]"); throw new StorageException(e.getMessage()); } } @@ -135,8 +130,8 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque logger.error("Unable to upload file to azure [" + e + "]"); throw new StorageException("Unable to upload file to azure", e); } - // Removes the parent directory and file - cleanupLocalFiles(source); + // Removes the file + cleanupLocalFiles(source, null); } /** @@ -229,10 +224,11 @@ private String getAzureFileAbsolutePath(Path file) { * {@inheritDoc} */ @Override - public void appendToFile(Path target, SequenceFile file) throws IOException{ + public void appendToFile(Path target, SequenceFile file) throws IOException { + IridaTemporaryFile iridaTemporaryFile = getTemporaryFile(file.getFile()); try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(getTemporaryFile(file.getFile())).getChannel()) { + try (FileChannel in = new FileInputStream(iridaTemporaryFile.getFile().toFile()).getChannel()) { for (long p = 0, l = in.size(); p < l; ) { p += in.transferTo(p, l - p, out); } @@ -242,6 +238,8 @@ public void appendToFile(Path target, SequenceFile file) throws IOException{ } catch (IOException e) { throw new StorageException("Could not open target file for writing", e); + } finally { + cleanupLocalFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 1600235bcc0..ace79ab4e50 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -21,6 +21,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.util.FileUtils; /** @@ -38,18 +39,19 @@ public IridaFileStorageLocalUtilityImpl(){ * {@inheritDoc} */ @Override - public File getTemporaryFile(Path file) { - File fileToProcess = null; - fileToProcess = file.toFile(); - return fileToProcess; + public IridaTemporaryFile getTemporaryFile(Path file) { + return new IridaTemporaryFile(file, null); } /** * {@inheritDoc} */ @Override - public void cleanupLocalFiles(Path path) { - logger.trace("File resides on local filesystem. Not cleaning up file [" + path.toString() + "]"); + public void cleanupLocalFiles(Path filePath, Path directoryPath) { + logger.trace("File resides on local filesystem. Not cleaning up file [" + filePath.toString() + "]"); + if(directoryPath != null) { + logger.trace("Directory resides on local filesystem. Not cleaning up directory [" + directoryPath.toString() + "]"); + } } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 272f335ea57..9aff502c587 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -8,6 +8,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import com.google.common.collect.Lists; @@ -22,16 +23,17 @@ public interface IridaFileStorageUtility { * Get a temporarry file from storage * * @param file The {@link Path} to the file - * @return {@link File} which was retrieved from path + * @return {@link IridaTemporaryFile} which includes the file and optional temporary directory */ - public File getTemporaryFile(Path file); + public IridaTemporaryFile getTemporaryFile(Path file); /** * Delete temporary file. * - * @param file The {@link Path} to the file + * @param filePath The {@link Path} to the file + * @param directoryPath The {@link Path} to the directory which has the file */ - public void cleanupLocalFiles(Path file); + public void cleanupLocalFiles(Path filePath, Path directoryPath); /** * Get file size diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java new file mode 100644 index 00000000000..46927df654e --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java @@ -0,0 +1,33 @@ +package ca.corefacility.bioinformatics.irida.ria.web.dto; + +import java.nio.file.Path; + +/** + * Used as a response for encapsulating a temporary file and it's directory + */ + +public class IridaTemporaryFile { + private Path filePath; + private Path directoryPath; + + public IridaTemporaryFile(Path filePath, Path directoryPath) { + this.filePath = filePath; + this.directoryPath = directoryPath; + } + + public Path getFile() { + return filePath; + } + + public void setFile(Path filePath) { + this.filePath = filePath; + } + + public Path getDirectoryPath() { + return directoryPath; + } + + public void setDirectoryPath(Path directoryPath) { + this.directoryPath = directoryPath; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index 05dafa7f8a7..0d7079c56fe 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -22,9 +22,11 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Controller for asynchronous requests for a {@link Sample} @@ -143,6 +145,7 @@ public ResponseEntity uploadAssemblies(@PathVariable Long sampleId, Mult UploadedAssembly uploadedAssembly = new UploadedAssembly(target); genomeAssemblyService.createAssemblyInSample(sample, uploadedAssembly); + IridaFiles.cleanupLocalFiles(null, temp); } return ResponseEntity.ok() .body(messageSource.getMessage("server.SampleFileUploader.success", @@ -163,9 +166,13 @@ public ResponseEntity uploadAssemblies(@PathVariable Long sampleId, Mult * @throws IOException Exception thrown if there is an error handling the file. */ private void createSequenceFilePairsInSample(List pair, Sample sample) throws IOException { - SequenceFile firstFile = createSequenceFile(pair.get(0)); - SequenceFile secondFile = createSequenceFile(pair.get(1)); + IridaTemporaryFile firstIridaTemporaryFile = createSequenceFile(pair.get(0)); + IridaTemporaryFile secondIridaTemporaryFile = createSequenceFile(pair.get(1)); + SequenceFile firstFile = new SequenceFile(firstIridaTemporaryFile.getFile()); + SequenceFile secondFile = new SequenceFile(secondIridaTemporaryFile.getFile()); sequencingObjectService.createSequencingObjectInSample(new SequenceFilePair(firstFile, secondFile), sample); + IridaFiles.cleanupLocalFiles(null, firstIridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalFiles(null, secondIridaTemporaryFile.getDirectoryPath()); } /** @@ -176,8 +183,10 @@ private void createSequenceFilePairsInSample(List pair, Sample sa * @throws IOException Exception thrown if there is an error handling the file. */ private void createSequenceFileInSample(MultipartFile file, Sample sample) throws IOException { - SequenceFile sequenceFile = createSequenceFile(file); + IridaTemporaryFile iridaTemporaryFile = createSequenceFile(file); + SequenceFile sequenceFile = new SequenceFile(iridaTemporaryFile.getFile()); sequencingObjectService.createSequencingObjectInSample(new SingleEndSequenceFile(sequenceFile), sample); + IridaFiles.cleanupLocalFiles(null, iridaTemporaryFile.getDirectoryPath()); } /** @@ -188,22 +197,24 @@ private void createSequenceFileInSample(MultipartFile file, Sample sample) throw * @throws IOException Exception thrown if there is an error handling the file. */ private void createFast5FileInSample(MultipartFile file, Sample sample) throws IOException { - SequenceFile sequenceFile = createSequenceFile(file); + IridaTemporaryFile iridaTemporaryFile = createSequenceFile(file); + SequenceFile sequenceFile = new SequenceFile(iridaTemporaryFile.getFile()); sequencingObjectService.createSequencingObjectInSample(new Fast5Object(sequenceFile), sample); + IridaFiles.cleanupLocalFiles(null, iridaTemporaryFile.getDirectoryPath()); } /** * Private method to move the sequence file into the correct directory and - * create the {@link SequenceFile} object. + * create the {@link IridaTemporaryFile} object. * * @param file {@link MultipartFile} sequence file uploaded. - * @return {@link SequenceFile} + * @return {@link IridaTemporaryFile} * @throws IOException Exception thrown if there is an error handling the file. */ - private SequenceFile createSequenceFile(MultipartFile file) throws IOException { + private IridaTemporaryFile createSequenceFile(MultipartFile file) throws IOException { Path temp = Files.createTempDirectory(null); Path target = temp.resolve(file.getOriginalFilename()); file.transferTo(target.toFile()); - return new SequenceFile(target); + return new IridaTemporaryFile(target, temp); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index d676014fd71..76e9c040303 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -20,6 +20,7 @@ public final class IridaFiles { public static void setIridaFileStorageUtility(IridaFileStorageUtility iridaFileStorageUtility) { IridaFiles.iridaFileStorageUtility = iridaFileStorageUtility; } + private IridaFiles() { } @@ -66,9 +67,14 @@ public static String getFileExtension(List files) th } /** - * Cleans up temporary downloaded files - * @param file The path to the file + * Cleans up temporary downloaded files. Can be used + * to clean up file and/or directory + * + * @param filePath The path to the file + * @param directoryPath The path to the directory which has the file */ - public static void cleanupLocalFiles(Path file) { iridaFileStorageUtility.cleanupLocalFiles(file); } + public static void cleanupLocalFiles(Path filePath, Path directoryPath) { + iridaFileStorageUtility.cleanupLocalFiles(filePath, directoryPath); + } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 9b667c25fad..8419f3c14bc 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -19,11 +19,13 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableList; @@ -35,6 +37,7 @@ public class SamplesAjaxControllerTest { private SamplesAjaxController controller; private SequencingObjectService sequencingObjectService; private GenomeAssemblyService genomeAssemblyService; + private IridaFileStorageLocalUtilityImpl iridaFileStorageUtility; /* TEST DATA @@ -58,6 +61,9 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); + iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); // Set up mocks From 4cf54992305acf71bf222471ec4fc5d0e5d4d5fe Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 31 Aug 2020 17:59:58 -0500 Subject: [PATCH 130/655] Fixed nullpointer --- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 4 +++- .../irida/processing/impl/unit/FastqcFileProcessorTest.java | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index ace79ab4e50..10b739642dd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -48,7 +48,9 @@ public IridaTemporaryFile getTemporaryFile(Path file) { */ @Override public void cleanupLocalFiles(Path filePath, Path directoryPath) { - logger.trace("File resides on local filesystem. Not cleaning up file [" + filePath.toString() + "]"); + if(filePath != null) { + logger.trace("File resides on local filesystem. Not cleaning up file [" + filePath.toString() + "]"); + } if(directoryPath != null) { logger.trace("Directory resides on local filesystem. Not cleaning up directory [" + directoryPath.toString() + "]"); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java index 2824eb6df1a..5e646746483 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/FastqcFileProcessorTest.java @@ -31,6 +31,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Tests for {@link FastqcFileProcessor}. @@ -56,6 +57,7 @@ public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); outputFileRepository = mock(AnalysisOutputFileRepository.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); fileProcessor = new FastqcFileProcessor(messageSource, sequenceFileRepository, outputFileRepository, iridaFileStorageUtility); } From 64984255444dfcf7490e7e32af61ca717a135937 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 31 Aug 2020 18:47:27 -0500 Subject: [PATCH 131/655] Removed unused imports --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 2 -- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 1 - .../irida/repositories/filesystem/IridaFileStorageUtility.java | 1 - 3 files changed, 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 2f044ba34af..145758fbbc2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -22,7 +21,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.util.FileUtils; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 10b739642dd..8ad002859a2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 9aff502c587..a8bf61db5f4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; From 05feefc11ed7c0e21deb78ad450036145ace7e5f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Sep 2020 13:07:05 -0500 Subject: [PATCH 132/655] Added a new method to cleanup local temporary files/directories and renamed previous method named cleanupLocalFiles. Updated logic in SamplesAjaxController to cleanup temp files/directories that are created. --- .../irida/model/sequenceFile/Fast5Object.java | 7 ++++- .../processing/impl/FastqcFileProcessor.java | 4 +-- .../processing/impl/GzipFileProcessor.java | 2 +- .../IridaFileStorageAzureUtilityImpl.java | 6 ++-- .../IridaFileStorageLocalUtilityImpl.java | 2 +- .../filesystem/IridaFileStorageUtility.java | 31 +++++++++++++++++-- .../web/files/ReferenceFileController.java | 7 ++--- .../web/samples/SamplesAjaxController.java | 12 ++++--- .../bioinformatics/irida/util/IridaFiles.java | 27 +++++++++++++--- .../samples/RESTSampleAssemblyController.java | 4 +-- .../RESTSampleSequenceFilesController.java | 13 +++----- 11 files changed, 81 insertions(+), 34 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index a5f81f28e04..2e8fa596162 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -11,6 +11,8 @@ import org.slf4j.LoggerFactory; import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; import liquibase.util.file.FilenameUtils; @@ -103,7 +105,10 @@ private Fast5Type setType(SequenceFile file) { try { String extension = FilenameUtils.getExtension(getFile().getFileName()); - if (file.isGzipped()) { + // Checks if file is where it should be before it checks if it is gzipped + if (IridaFiles.fileExists(file.getFile()) && file.isGzipped()) { + type = Fast5Object.Fast5Type.ZIPPED; + } else if (extension.equals("gz")) { type = Fast5Object.Fast5Type.ZIPPED; } else if (extension.equals("fast5")) { type = Fast5Object.Fast5Type.SINGLE; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index a09309f648d..c33f7dd0c5b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -148,8 +148,8 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx logger.error("Unable to create temporary directory ", e); throw new StorageException("Unable to create temporary directory", e); } finally { - IridaFiles.cleanupLocalFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); - IridaFiles.cleanupLocalFiles(null, outputDirectory); + IridaFiles.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalTemporaryFiles(null, outputDirectory); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index 3a44d8bb986..fdd89e2ecfb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -152,7 +152,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc logger.error("Unable to create temporary directory", e); throw new StorageException("Unable to create temporary directory"); } finally { - IridaFiles.cleanupLocalFiles(null, targetDirectory); + IridaFiles.cleanupLocalTemporaryFiles(null, targetDirectory); } } } catch (Exception e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 145758fbbc2..edd2acb6d26 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -74,7 +74,7 @@ public IridaTemporaryFile getTemporaryFile(Path file) { * {@inheritDoc} */ @Override - public void cleanupLocalFiles(Path filePath, Path directoryPath) { + public void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPath) { try { if(filePath != null && Files.isRegularFile(filePath)) { logger.trace("Cleaning up temporary file downloaded from azure [" + filePath.toString() + "]"); @@ -128,8 +128,6 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque logger.error("Unable to upload file to azure [" + e + "]"); throw new StorageException("Unable to upload file to azure", e); } - // Removes the file - cleanupLocalFiles(source, null); } /** @@ -237,7 +235,7 @@ public void appendToFile(Path target, SequenceFile file) throws IOException { } catch (IOException e) { throw new StorageException("Could not open target file for writing", e); } finally { - cleanupLocalFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); + cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 8ad002859a2..64a69d8eacc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -46,7 +46,7 @@ public IridaTemporaryFile getTemporaryFile(Path file) { * {@inheritDoc} */ @Override - public void cleanupLocalFiles(Path filePath, Path directoryPath) { + public void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPath) { if(filePath != null) { logger.trace("File resides on local filesystem. Not cleaning up file [" + filePath.toString() + "]"); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index a8bf61db5f4..3e93a09c7c2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -2,9 +2,14 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; @@ -16,6 +21,8 @@ */ public interface IridaFileStorageUtility { + Logger logger = LoggerFactory.getLogger(IridaFileStorageUtility.class); + //Valid file extensions for sample file concatenation public static final List VALID_CONCATENATION_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** @@ -27,12 +34,12 @@ public interface IridaFileStorageUtility { public IridaTemporaryFile getTemporaryFile(Path file); /** - * Delete temporary file. + * Delete temporary downloaded file and/or directory. * * @param filePath The {@link Path} to the file * @param directoryPath The {@link Path} to the directory which has the file */ - public void cleanupLocalFiles(Path filePath, Path directoryPath); + public void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPath); /** * Get file size @@ -114,4 +121,24 @@ public interface IridaFileStorageUtility { * @throws IOException if the files have different or invalid extensions */ public String getFileExtension(List sequencingObjects) throws IOException; + + /** + * Delete temporary file and/or directory. + * + * @param filePath The {@link Path} to the file + * @param directoryPath The {@link Path} to the directory which has the file + */ + public static void cleanupLocalTemporaryFiles(Path filePath, Path directoryPath) { + try { + if(filePath != null) { + Files.delete(filePath); + } + if(directoryPath != null) { + org.apache.commons.io.FileUtils.deleteDirectory(directoryPath.toFile()); + } + } catch (IOException e) { + logger.error("Unable to clean up local files/directories", e); + throw new StorageException(e.getMessage()); + } + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java index 8ff16343e33..1cc6fcc30b5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java @@ -33,6 +33,7 @@ import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Controller for all {@link ReferenceFile} related views @@ -105,8 +106,7 @@ public void downloadReferenceFile(@PathVariable Long fileId, projectService.addReferenceFileToProject(project, referenceFile); // Clean up temporary files - Files.deleteIfExists(target); - Files.deleteIfExists(temp); + IridaFiles.cleanupLocalTemporaryFiles(target, temp); } } catch (final UnsupportedReferenceFileContentError e) { logger.error("User uploaded a reference file that biojava couldn't parse as DNA.", e); @@ -156,8 +156,7 @@ public Map addIndependentReferenceFile( } // Clean up temporary files - Files.deleteIfExists(target); - Files.deleteIfExists(temp); + IridaFiles.cleanupLocalTemporaryFiles(target, temp); return ImmutableMap.of("uploaded-file-id", referenceFile.getId(), "uploaded-file-name", referenceFile.getLabel()); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index 0d7079c56fe..5f6dc10445b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -145,7 +145,9 @@ public ResponseEntity uploadAssemblies(@PathVariable Long sampleId, Mult UploadedAssembly uploadedAssembly = new UploadedAssembly(target); genomeAssemblyService.createAssemblyInSample(sample, uploadedAssembly); - IridaFiles.cleanupLocalFiles(null, temp); + + // Clean up temporary files + IridaFiles.cleanupLocalTemporaryFiles(target, temp); } return ResponseEntity.ok() .body(messageSource.getMessage("server.SampleFileUploader.success", @@ -171,8 +173,8 @@ private void createSequenceFilePairsInSample(List pair, Sample sa SequenceFile firstFile = new SequenceFile(firstIridaTemporaryFile.getFile()); SequenceFile secondFile = new SequenceFile(secondIridaTemporaryFile.getFile()); sequencingObjectService.createSequencingObjectInSample(new SequenceFilePair(firstFile, secondFile), sample); - IridaFiles.cleanupLocalFiles(null, firstIridaTemporaryFile.getDirectoryPath()); - IridaFiles.cleanupLocalFiles(null, secondIridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalTemporaryFiles(firstIridaTemporaryFile.getFile(), firstIridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalTemporaryFiles(secondIridaTemporaryFile.getFile(), secondIridaTemporaryFile.getDirectoryPath()); } /** @@ -186,7 +188,7 @@ private void createSequenceFileInSample(MultipartFile file, Sample sample) throw IridaTemporaryFile iridaTemporaryFile = createSequenceFile(file); SequenceFile sequenceFile = new SequenceFile(iridaTemporaryFile.getFile()); sequencingObjectService.createSequencingObjectInSample(new SingleEndSequenceFile(sequenceFile), sample); - IridaFiles.cleanupLocalFiles(null, iridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalTemporaryFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); } /** @@ -200,7 +202,7 @@ private void createFast5FileInSample(MultipartFile file, Sample sample) throws I IridaTemporaryFile iridaTemporaryFile = createSequenceFile(file); SequenceFile sequenceFile = new SequenceFile(iridaTemporaryFile.getFile()); sequencingObjectService.createSequencingObjectInSample(new Fast5Object(sequenceFile), sample); - IridaFiles.cleanupLocalFiles(null, iridaTemporaryFile.getDirectoryPath()); + IridaFiles.cleanupLocalTemporaryFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 76e9c040303..7401455a403 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -67,14 +67,33 @@ public static String getFileExtension(List files) th } /** - * Cleans up temporary downloaded files. Can be used - * to clean up file and/or directory + * Cleans up temporary downloaded files. * * @param filePath The path to the file * @param directoryPath The path to the directory which has the file */ - public static void cleanupLocalFiles(Path filePath, Path directoryPath) { - iridaFileStorageUtility.cleanupLocalFiles(filePath, directoryPath); + public static void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPath) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(filePath, directoryPath); + } + + /** + * Cleans up temporary files. + * + * @param filePath The path to the file + * @param directoryPath The path to the directory which has the file + */ + public static void cleanupLocalTemporaryFiles(Path filePath, Path directoryPath) { + IridaFileStorageUtility.cleanupLocalTemporaryFiles(filePath, directoryPath); + } + + /** + * Checks if the file exists + * + * @param file The path to the file + * @return if file exists or not + */ + public static boolean fileExists(Path file) { + return iridaFileStorageUtility.fileExists(file); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java index 140c9cc279b..c4346af2d59 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java @@ -32,6 +32,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResourceCollection; import ca.corefacility.bioinformatics.irida.web.controller.api.RESTAnalysisSubmissionController; import ca.corefacility.bioinformatics.irida.web.controller.api.RESTGenericController; @@ -182,8 +183,7 @@ public ModelMap addNewAssemblyToSample(@PathVariable Long sampleId, @RequestPart } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - Files.deleteIfExists(target); - Files.deleteIfExists(temp); + IridaFiles.cleanupLocalTemporaryFiles(target, temp); } return modelMap; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index bda188e9da5..a0f67252fe2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -34,6 +34,7 @@ import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResourceCollection; import ca.corefacility.bioinformatics.irida.web.assembler.resource.RootResource; import ca.corefacility.bioinformatics.irida.web.assembler.resource.sequencefile.SequenceFileResource; @@ -472,8 +473,7 @@ public ModelMap addNewSequenceFileToSample(@PathVariable Long sampleId, @Request } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - Files.deleteIfExists(target); - Files.deleteIfExists(temp); + IridaFiles.cleanupLocalTemporaryFiles(target, temp); } // respond to the client @@ -584,8 +584,7 @@ public ModelMap addNewFast5FileToSample(@PathVariable Long sampleId, @RequestPar } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - Files.deleteIfExists(target); - Files.deleteIfExists(temp); + IridaFiles.cleanupLocalTemporaryFiles(target, temp); } // respond to the client @@ -687,10 +686,8 @@ public ModelMap addNewSequenceFilePairToSample(@PathVariable Long sampleId, } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - Files.deleteIfExists(target1); - Files.deleteIfExists(temp1); - Files.deleteIfExists(target2); - Files.deleteIfExists(temp2); + IridaFiles.cleanupLocalTemporaryFiles(target1, temp1); + IridaFiles.cleanupLocalTemporaryFiles(target2, temp2); } // respond to the client From 8978508ddb1c7efcf74bea343516ea5fdb24b867 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Sep 2020 13:46:02 -0500 Subject: [PATCH 133/655] Fixed logic so if the instance is using local storage it will clean up the file in a diretory if it exists --- .../irida/repositories/filesystem/IridaFileStorageUtility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 3e93a09c7c2..a9494653ad9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -131,7 +131,7 @@ public interface IridaFileStorageUtility { public static void cleanupLocalTemporaryFiles(Path filePath, Path directoryPath) { try { if(filePath != null) { - Files.delete(filePath); + Files.deleteIfExists(filePath); } if(directoryPath != null) { org.apache.commons.io.FileUtils.deleteDirectory(directoryPath.toFile()); From 7275c81aec8be26d33a80a302ad2a5a2484fd91a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Sep 2020 15:33:06 -0500 Subject: [PATCH 134/655] Updated concatenator classes to return an iridaconcatenatortemporyfile pojo which contains the sequencing object and temporary file directory path which can be cleaned up in the sequencingobjectservice. Updated tests --- .../SequencingObjectConcatenator.java | 5 +- .../impl/SequenceFilePairConcatenator.java | 10 ++-- .../SingleEndSequenceFileConcatenator.java | 9 ++-- .../dto/IridaConcatenatorTemporaryFile.java | 46 +++++++++++++++++++ .../impl/SequencingObjectServiceImpl.java | 10 +++- .../SequenceFilePairConcatenatorTest.java | 4 +- ...SingleEndSequenceFileConcatenatorTest.java | 13 ++++-- 7 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index 2294c83b3b2..6db67d03497 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -2,6 +2,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import java.util.List; @@ -18,10 +19,10 @@ public abstract class SequencingObjectConcatenator toConcatenate, String filename) + public abstract IridaConcatenatorTemporaryFile concatenateFiles(List toConcatenate, String filename) throws ConcatenateException; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index ade211d4db0..dc700254e91 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -6,6 +6,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; @@ -28,8 +29,8 @@ public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtil /** * {@inheritDoc} */ - @Override - public SequenceFilePair concatenateFiles(List toConcatenate, String filename) + + public IridaConcatenatorTemporaryFile concatenateFiles(List toConcatenate, String filename) throws ConcatenateException { String extension = null; @@ -43,11 +44,12 @@ public SequenceFilePair concatenateFiles(List toConc String forwardName = filename + "_R1." + extension; String reverseName = filename + "_R2." + extension; + Path tempDirectory; Path forwardFile; Path reverseFile; try { // create a temp directory for the new files - Path tempDirectory = Files.createTempDirectory(null); + tempDirectory = Files.createTempDirectory(null); forwardFile = tempDirectory.resolve(forwardName); reverseFile = tempDirectory.resolve(reverseName); @@ -82,6 +84,6 @@ public SequenceFilePair concatenateFiles(List toConc // create the new pair SequenceFilePair sequenceFilePair = new SequenceFilePair(forward, reverse); - return sequenceFilePair; + return new IridaConcatenatorTemporaryFile(null, tempDirectory, sequenceFilePair); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java index 67b1d3cc205..30febea5799 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenator.java @@ -6,6 +6,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; @@ -29,10 +30,10 @@ public SingleEndSequenceFileConcatenator(IridaFileStorageUtility iridaFileStorag * {@inheritDoc} */ @Override - public SingleEndSequenceFile concatenateFiles(List toConcatenate, String filename) + public IridaConcatenatorTemporaryFile concatenateFiles(List toConcatenate, String filename) throws ConcatenateException { Path tempFile; - + Path tempDirectory; String extension = null; try { @@ -44,7 +45,7 @@ public SingleEndSequenceFile concatenateFiles(List t filename = filename + "." + extension; try { // create a temp directory and temp file - Path tempDirectory = Files.createTempDirectory(null); + tempDirectory = Files.createTempDirectory(null); tempFile = tempDirectory.resolve(filename); @@ -72,7 +73,7 @@ public SingleEndSequenceFile concatenateFiles(List t SingleEndSequenceFile seqObject = new SingleEndSequenceFile(forward); - return seqObject; + return new IridaConcatenatorTemporaryFile(null, tempDirectory, seqObject); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java new file mode 100644 index 00000000000..78a4437c09d --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java @@ -0,0 +1,46 @@ +package ca.corefacility.bioinformatics.irida.ria.web.dto; + +import java.nio.file.Path; + +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; + +/** + * Used as a response for encapsulating a temporary file and it's directory + * for a concatenator object + */ + +public class IridaConcatenatorTemporaryFile { + private Path filePath; + private Path directoryPath; + private SequencingObject sequencingObject; + + public IridaConcatenatorTemporaryFile(Path filePath, Path directoryPath, SequencingObject sequencingObject) { + this.filePath = filePath; + this.directoryPath = directoryPath; + this.sequencingObject = sequencingObject; + } + + public Path getFilePath() { + return filePath; + } + + public void setFilePath(Path filePath) { + this.filePath = filePath; + } + + public Path getDirectoryPath() { + return directoryPath; + } + + public void setDirectoryPath(Path directoryPath) { + this.directoryPath = directoryPath; + } + + public SequencingObject getSequencingObject() { + return sequencingObject; + } + + public void setSequencingObject(SequencingObject sequencingObject) { + this.sequencingObject = sequencingObject; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index 27d62293fb8..e1b1db45e27 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -18,7 +18,10 @@ import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.repositories.specification.SampleSequencingObjectSpecification; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import com.google.common.collect.ImmutableMap; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; @@ -250,8 +253,9 @@ public SampleSequencingObjectJoin concatenateSequences(List to SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory .getConcatenator(toJoin, iridaFileStorageUtility); - SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); - + IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concatenator.concatenateFiles(toJoin, filename); + SequencingObject concatenated = iridaConcatenatorTemporaryFile.getSequencingObject(); + SampleSequencingObjectJoin created = createSequencingObjectInSample(concatenated, targetSample); concatenationRepository.save(new SequenceConcatenation(created.getObject(), toJoin)); @@ -263,6 +267,8 @@ public SampleSequencingObjectJoin concatenateSequences(List to } } + IridaFiles.cleanupLocalTemporaryFiles(null, iridaConcatenatorTemporaryFile.getDirectoryPath()); + return created; } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index ba26c80d185..790d1591f80 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; @@ -50,7 +51,8 @@ public void testConcatenateFiles() throws IOException, ConcatenateException { SequenceFilePair f1 = new SequenceFilePair(original1, original2); SequenceFilePair f2 = new SequenceFilePair(original3, original4); - SequenceFilePair concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + SequenceFilePair concatenateFiles = (SequenceFilePair) iridaConcatenatorTemporaryFile.getSequencingObject(); SequenceFile forward = concatenateFiles.getForwardSequenceFile(); SequenceFile reverse = concatenateFiles.getReverseSequenceFile(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 5652308314c..8e4fc949198 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -5,6 +5,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; @@ -47,7 +48,8 @@ public void testConcatenateFiles() throws IOException, ConcatenateException { SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); SequenceFile newSeqFile = concatenateFiles.getSequenceFile(); @@ -74,7 +76,8 @@ public void testConcatenateFilesZipped() throws IOException, ConcatenateExceptio SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); SequenceFile newSeqFile = concatenateFiles.getSequenceFile(); @@ -97,7 +100,8 @@ public void testConcatenateDifferentFileTypes() throws IOException, ConcatenateE SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); } @Test(expected = ConcatenateException.class) @@ -110,7 +114,8 @@ public void testConcatenateBadExtension() throws IOException, ConcatenateExcepti SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); + SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); } private SequenceFile createSequenceFile(String name, String extension) throws IOException { From fa0aca42498ea6647309db43637647e544db2b81 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 2 Sep 2020 10:13:30 -0500 Subject: [PATCH 135/655] Updated gzipfileprocessor to cleanup temporary local files and directories if the removeTemporaryFiles variable is set to true (defaults to true for autowired constructor but second constructor has an extra variable which is used by the gzipfileprocessortest to not delete the temporary files as the files are cleaned up once the test has finished running). Updated logger text. --- .../irida/processing/impl/GzipFileProcessor.java | 12 +++++++++--- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 2 +- .../filesystem/IridaFileStorageUtility.java | 2 ++ .../processing/impl/unit/GzipFileProcessorTest.java | 4 ++-- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index fdd89e2ecfb..49bbe153734 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -41,17 +41,20 @@ public class GzipFileProcessor implements FileProcessor { private boolean disableFileProcessor = false; private boolean removeCompressedFile; private IridaFileStorageUtility iridaFileStorageUtility; + private boolean removeTemporaryFiles; @Autowired public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository) { this.sequenceFileRepository = sequenceFileRepository; removeCompressedFile = false; + removeTemporaryFiles = true; } - public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles, IridaFileStorageUtility iridaFileStorageUtility) { + public GzipFileProcessor(final SequenceFileRepository sequenceFileRepository, Boolean removeCompressedFiles, IridaFileStorageUtility iridaFileStorageUtility, Boolean removeTemporaryFiles) { this.sequenceFileRepository = sequenceFileRepository; this.removeCompressedFile = removeCompressedFiles; this.iridaFileStorageUtility = iridaFileStorageUtility; + this.removeTemporaryFiles = removeTemporaryFiles; } /** @@ -119,6 +122,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); Path targetDirectory = null; + Path target = null; try { targetDirectory = Files.createTempDirectory(null); @@ -126,7 +130,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { logger.trace("Handling gzip compressed file."); - Path target = targetDirectory.resolve(nameWithoutExtension); + target = targetDirectory.resolve(nameWithoutExtension); logger.debug("Target directory is [" + targetDirectory + "]"); logger.debug("Writing uncompressed file to [" + target + "]"); @@ -152,7 +156,9 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc logger.error("Unable to create temporary directory", e); throw new StorageException("Unable to create temporary directory"); } finally { - IridaFiles.cleanupLocalTemporaryFiles(null, targetDirectory); + if(removeTemporaryFiles) { + IridaFiles.cleanupLocalTemporaryFiles(target, targetDirectory); + } } } } catch (Exception e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index edd2acb6d26..333e2f976aa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -87,7 +87,7 @@ public void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPa try { if(directoryPath != null && Files.isDirectory(directoryPath)) { - logger.trace("Cleaning up temporary directory [" + directoryPath.toString() + "]"); + logger.trace("Cleaning up temporary directory created for azure temporary file [" + directoryPath.toString() + "]"); org.apache.commons.io.FileUtils.deleteDirectory(directoryPath.toFile()); } } catch (IOException e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index a9494653ad9..89e39c347fd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -131,9 +131,11 @@ public interface IridaFileStorageUtility { public static void cleanupLocalTemporaryFiles(Path filePath, Path directoryPath) { try { if(filePath != null) { + logger.trace("Cleaning up temporary file: [" + filePath.toString() + "]"); Files.deleteIfExists(filePath); } if(directoryPath != null) { + logger.trace("Cleaning up temporary directory: [" + directoryPath.toString() + "]"); org.apache.commons.io.FileUtils.deleteDirectory(directoryPath.toFile()); } } catch (IOException e) { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index a3b3616928e..294cb3ac0b7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -44,7 +44,7 @@ public class GzipFileProcessorTest { public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageUtility); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageUtility, Boolean.FALSE); IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @@ -67,7 +67,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageUtility); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageUtility, Boolean.FALSE); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference From 8bd9018d94882d33a9f10ab0a3007f64ec632e0b Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 2 Sep 2020 10:21:17 -0500 Subject: [PATCH 136/655] Updated method comment --- .../irida/repositories/filesystem/IridaFileStorageUtility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 89e39c347fd..63ea808f07a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -123,7 +123,7 @@ public interface IridaFileStorageUtility { public String getFileExtension(List sequencingObjects) throws IOException; /** - * Delete temporary file and/or directory. + * Delete local temporary file and/or directory. * * @param filePath The {@link Path} to the file * @param directoryPath The {@link Path} to the directory which has the file From a3254d00f733d0780c7d91896177e6f5da2b883c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 2 Sep 2020 10:27:39 -0500 Subject: [PATCH 137/655] Removed @Responsebody annotation for test methods as it is a restcontroller and the contentnegotiator returns the data as json --- .../irida/ria/web/analysis/AnalysesTableAjaxController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java index e0a5a49cac2..f61f457008c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysesTableAjaxController.java @@ -74,7 +74,6 @@ public AnalysesTableAjaxController(AnalysisSubmissionService analysisSubmissionS * @return {@link List} of {@link AnalysisState} */ @RequestMapping("/states") - @ResponseBody public List getAnalysisStates(Locale locale) { List states = Arrays.asList(AnalysisState.values()); return states.stream() @@ -90,7 +89,6 @@ public List getAnalysisStates(Locale locale) { * @return an internationalized list of analysis type names */ @RequestMapping("/types") - @ResponseBody public List getWorkflowTypes(Locale locale) { Set types = iridaWorkflowsService.getRegisteredWorkflowTypes(); return types.stream() From 1dbdb47af3eedeee57410916380b562e92a85c41 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 9 Sep 2020 08:03:25 -0500 Subject: [PATCH 138/655] Reverted changes to clean up local temporary files. Updated cleanup downloaded local files method to accept an IridaTemporaryFile object instead of paths. --- .../SequencingObjectConcatenator.java | 5 +- .../impl/SequenceFilePairConcatenator.java | 10 ++- .../SingleEndSequenceFileConcatenator.java | 8 +-- .../processing/impl/FastqcFileProcessor.java | 9 ++- .../processing/impl/GzipFileProcessor.java | 68 +++++++------------ .../IridaFileStorageAzureUtilityImpl.java | 28 +++++--- .../IridaFileStorageLocalUtilityImpl.java | 10 +-- .../filesystem/IridaFileStorageUtility.java | 33 +-------- .../remote/impl/RemoteRepositoryImpl.java | 1 + .../dto/IridaConcatenatorTemporaryFile.java | 46 ------------- .../web/files/ReferenceFileController.java | 7 +- .../web/samples/SamplesAjaxController.java | 29 +++----- .../impl/SequencingObjectServiceImpl.java | 8 +-- .../bioinformatics/irida/util/IridaFiles.java | 21 +----- .../RESTProjectSamplesController.java | 1 + .../projects/RESTProjectUsersController.java | 5 +- .../samples/RESTSampleAssemblyController.java | 4 +- .../RESTSampleSequenceFilesController.java | 13 ++-- .../SequenceFilePairConcatenatorTest.java | 4 +- ...SingleEndSequenceFileConcatenatorTest.java | 13 ++-- .../impl/unit/GzipFileProcessorTest.java | 4 +- .../samples/SamplesAjaxControllerTest.java | 4 -- 22 files changed, 104 insertions(+), 227 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java index 6db67d03497..2294c83b3b2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/SequencingObjectConcatenator.java @@ -2,7 +2,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import java.util.List; @@ -19,10 +18,10 @@ public abstract class SequencingObjectConcatenator toConcatenate, String filename) + public abstract Type concatenateFiles(List toConcatenate, String filename) throws ConcatenateException; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index dc700254e91..ade211d4db0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -6,7 +6,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.concatenate.SequencingObjectConcatenator; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import java.io.IOException; @@ -29,8 +28,8 @@ public SequenceFilePairConcatenator(IridaFileStorageUtility iridaFileStorageUtil /** * {@inheritDoc} */ - - public IridaConcatenatorTemporaryFile concatenateFiles(List toConcatenate, String filename) + @Override + public SequenceFilePair concatenateFiles(List toConcatenate, String filename) throws ConcatenateException { String extension = null; @@ -44,12 +43,11 @@ public IridaConcatenatorTemporaryFile concatenateFiles(List toConcatenate, String filename) + public SingleEndSequenceFile concatenateFiles(List toConcatenate, String filename) throws ConcatenateException { Path tempFile; - Path tempDirectory; String extension = null; try { @@ -45,7 +43,7 @@ public IridaConcatenatorTemporaryFile concatenateFiles(List VALID_CONCATENATION_EXTENSIONS = Lists.newArrayList("fastq", "fastq.gz"); /** @@ -36,10 +29,9 @@ public interface IridaFileStorageUtility { /** * Delete temporary downloaded file and/or directory. * - * @param filePath The {@link Path} to the file - * @param directoryPath The {@link Path} to the directory which has the file + * @param iridaTemporaryFile The {@link IridaTemporaryFile} object which includes the file path and/or directory path */ - public void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPath); + public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTemporaryFile); /** * Get file size @@ -122,25 +114,4 @@ public interface IridaFileStorageUtility { */ public String getFileExtension(List sequencingObjects) throws IOException; - /** - * Delete local temporary file and/or directory. - * - * @param filePath The {@link Path} to the file - * @param directoryPath The {@link Path} to the directory which has the file - */ - public static void cleanupLocalTemporaryFiles(Path filePath, Path directoryPath) { - try { - if(filePath != null) { - logger.trace("Cleaning up temporary file: [" + filePath.toString() + "]"); - Files.deleteIfExists(filePath); - } - if(directoryPath != null) { - logger.trace("Cleaning up temporary directory: [" + directoryPath.toString() + "]"); - org.apache.commons.io.FileUtils.deleteDirectory(directoryPath.toFile()); - } - } catch (IOException e) { - logger.error("Unable to clean up local files/directories", e); - throw new StorageException(e.getMessage()); - } - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java index a4bee9ad3e3..62edd9adb6e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/remote/impl/RemoteRepositoryImpl.java @@ -96,6 +96,7 @@ public List list(String uri, RemoteAPI remoteAPI) { public boolean getServiceStatus(RemoteAPI remoteAPI) { OAuthTokenRestTemplate restTemplate = new OAuthTokenRestTemplate(tokenService, remoteAPI); ResponseEntity forEntity = restTemplate.getForEntity(remoteAPI.getServiceURI(), String.class); + return forEntity.getStatusCode() == HttpStatus.OK; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java deleted file mode 100644 index 78a4437c09d..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaConcatenatorTemporaryFile.java +++ /dev/null @@ -1,46 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.dto; - -import java.nio.file.Path; - -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; - -/** - * Used as a response for encapsulating a temporary file and it's directory - * for a concatenator object - */ - -public class IridaConcatenatorTemporaryFile { - private Path filePath; - private Path directoryPath; - private SequencingObject sequencingObject; - - public IridaConcatenatorTemporaryFile(Path filePath, Path directoryPath, SequencingObject sequencingObject) { - this.filePath = filePath; - this.directoryPath = directoryPath; - this.sequencingObject = sequencingObject; - } - - public Path getFilePath() { - return filePath; - } - - public void setFilePath(Path filePath) { - this.filePath = filePath; - } - - public Path getDirectoryPath() { - return directoryPath; - } - - public void setDirectoryPath(Path directoryPath) { - this.directoryPath = directoryPath; - } - - public SequencingObject getSequencingObject() { - return sequencingObject; - } - - public void setSequencingObject(SequencingObject sequencingObject) { - this.sequencingObject = sequencingObject; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java index 1cc6fcc30b5..8ff16343e33 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/ReferenceFileController.java @@ -33,7 +33,6 @@ import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Controller for all {@link ReferenceFile} related views @@ -106,7 +105,8 @@ public void downloadReferenceFile(@PathVariable Long fileId, projectService.addReferenceFileToProject(project, referenceFile); // Clean up temporary files - IridaFiles.cleanupLocalTemporaryFiles(target, temp); + Files.deleteIfExists(target); + Files.deleteIfExists(temp); } } catch (final UnsupportedReferenceFileContentError e) { logger.error("User uploaded a reference file that biojava couldn't parse as DNA.", e); @@ -156,7 +156,8 @@ public Map addIndependentReferenceFile( } // Clean up temporary files - IridaFiles.cleanupLocalTemporaryFiles(target, temp); + Files.deleteIfExists(target); + Files.deleteIfExists(temp); return ImmutableMap.of("uploaded-file-id", referenceFile.getId(), "uploaded-file-name", referenceFile.getLabel()); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index 5f6dc10445b..05dafa7f8a7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -22,11 +22,9 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Controller for asynchronous requests for a {@link Sample} @@ -145,9 +143,6 @@ public ResponseEntity uploadAssemblies(@PathVariable Long sampleId, Mult UploadedAssembly uploadedAssembly = new UploadedAssembly(target); genomeAssemblyService.createAssemblyInSample(sample, uploadedAssembly); - - // Clean up temporary files - IridaFiles.cleanupLocalTemporaryFiles(target, temp); } return ResponseEntity.ok() .body(messageSource.getMessage("server.SampleFileUploader.success", @@ -168,13 +163,9 @@ public ResponseEntity uploadAssemblies(@PathVariable Long sampleId, Mult * @throws IOException Exception thrown if there is an error handling the file. */ private void createSequenceFilePairsInSample(List pair, Sample sample) throws IOException { - IridaTemporaryFile firstIridaTemporaryFile = createSequenceFile(pair.get(0)); - IridaTemporaryFile secondIridaTemporaryFile = createSequenceFile(pair.get(1)); - SequenceFile firstFile = new SequenceFile(firstIridaTemporaryFile.getFile()); - SequenceFile secondFile = new SequenceFile(secondIridaTemporaryFile.getFile()); + SequenceFile firstFile = createSequenceFile(pair.get(0)); + SequenceFile secondFile = createSequenceFile(pair.get(1)); sequencingObjectService.createSequencingObjectInSample(new SequenceFilePair(firstFile, secondFile), sample); - IridaFiles.cleanupLocalTemporaryFiles(firstIridaTemporaryFile.getFile(), firstIridaTemporaryFile.getDirectoryPath()); - IridaFiles.cleanupLocalTemporaryFiles(secondIridaTemporaryFile.getFile(), secondIridaTemporaryFile.getDirectoryPath()); } /** @@ -185,10 +176,8 @@ private void createSequenceFilePairsInSample(List pair, Sample sa * @throws IOException Exception thrown if there is an error handling the file. */ private void createSequenceFileInSample(MultipartFile file, Sample sample) throws IOException { - IridaTemporaryFile iridaTemporaryFile = createSequenceFile(file); - SequenceFile sequenceFile = new SequenceFile(iridaTemporaryFile.getFile()); + SequenceFile sequenceFile = createSequenceFile(file); sequencingObjectService.createSequencingObjectInSample(new SingleEndSequenceFile(sequenceFile), sample); - IridaFiles.cleanupLocalTemporaryFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); } /** @@ -199,24 +188,22 @@ private void createSequenceFileInSample(MultipartFile file, Sample sample) throw * @throws IOException Exception thrown if there is an error handling the file. */ private void createFast5FileInSample(MultipartFile file, Sample sample) throws IOException { - IridaTemporaryFile iridaTemporaryFile = createSequenceFile(file); - SequenceFile sequenceFile = new SequenceFile(iridaTemporaryFile.getFile()); + SequenceFile sequenceFile = createSequenceFile(file); sequencingObjectService.createSequencingObjectInSample(new Fast5Object(sequenceFile), sample); - IridaFiles.cleanupLocalTemporaryFiles(iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); } /** * Private method to move the sequence file into the correct directory and - * create the {@link IridaTemporaryFile} object. + * create the {@link SequenceFile} object. * * @param file {@link MultipartFile} sequence file uploaded. - * @return {@link IridaTemporaryFile} + * @return {@link SequenceFile} * @throws IOException Exception thrown if there is an error handling the file. */ - private IridaTemporaryFile createSequenceFile(MultipartFile file) throws IOException { + private SequenceFile createSequenceFile(MultipartFile file) throws IOException { Path temp = Files.createTempDirectory(null); Path target = temp.resolve(file.getOriginalFilename()); file.transferTo(target.toFile()); - return new IridaTemporaryFile(target, temp); + return new SequenceFile(target); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index e1b1db45e27..5585ef50458 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -18,9 +18,7 @@ import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.repositories.specification.SampleSequencingObjectSpecification; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableMap; import org.springframework.beans.factory.annotation.Autowired; @@ -253,9 +251,7 @@ public SampleSequencingObjectJoin concatenateSequences(List to SequencingObjectConcatenator concatenator = SequencingObjectConcatenatorFactory .getConcatenator(toJoin, iridaFileStorageUtility); - IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concatenator.concatenateFiles(toJoin, filename); - SequencingObject concatenated = iridaConcatenatorTemporaryFile.getSequencingObject(); - + SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); SampleSequencingObjectJoin created = createSequencingObjectInSample(concatenated, targetSample); concatenationRepository.save(new SequenceConcatenation(created.getObject(), toJoin)); @@ -267,8 +263,6 @@ public SampleSequencingObjectJoin concatenateSequences(List to } } - IridaFiles.cleanupLocalTemporaryFiles(null, iridaConcatenatorTemporaryFile.getDirectoryPath()); - return created; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 7401455a403..3c985468fa4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -7,6 +7,7 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; /** * Static class which has file object operations that require @@ -66,26 +67,6 @@ public static String getFileExtension(List files) th return iridaFileStorageUtility.getFileExtension(files); } - /** - * Cleans up temporary downloaded files. - * - * @param filePath The path to the file - * @param directoryPath The path to the directory which has the file - */ - public static void cleanupDownloadedLocalTemporaryFiles(Path filePath, Path directoryPath) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(filePath, directoryPath); - } - - /** - * Cleans up temporary files. - * - * @param filePath The path to the file - * @param directoryPath The path to the directory which has the file - */ - public static void cleanupLocalTemporaryFiles(Path filePath, Path directoryPath) { - IridaFileStorageUtility.cleanupLocalTemporaryFiles(filePath, directoryPath); - } - /** * Checks if the file exists * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java index 3af0471cb21..461ed04d3f5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectSamplesController.java @@ -36,6 +36,7 @@ import ca.corefacility.bioinformatics.irida.web.controller.api.samples.RESTSampleSequenceFilesController; import com.google.common.net.HttpHeaders; + import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java index 0f02958d928..644b6c63a92 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/projects/RESTProjectUsersController.java @@ -12,7 +12,10 @@ import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import ca.corefacility.bioinformatics.irida.exceptions.ProjectWithoutOwnerException; import ca.corefacility.bioinformatics.irida.model.enums.ProjectRole; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java index c4346af2d59..140c9cc279b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleAssemblyController.java @@ -32,7 +32,6 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResourceCollection; import ca.corefacility.bioinformatics.irida.web.controller.api.RESTAnalysisSubmissionController; import ca.corefacility.bioinformatics.irida.web.controller.api.RESTGenericController; @@ -183,7 +182,8 @@ public ModelMap addNewAssemblyToSample(@PathVariable Long sampleId, @RequestPart } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - IridaFiles.cleanupLocalTemporaryFiles(target, temp); + Files.deleteIfExists(target); + Files.deleteIfExists(temp); } return modelMap; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index a0f67252fe2..bda188e9da5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -34,7 +34,6 @@ import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.SequencingRunService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResourceCollection; import ca.corefacility.bioinformatics.irida.web.assembler.resource.RootResource; import ca.corefacility.bioinformatics.irida.web.assembler.resource.sequencefile.SequenceFileResource; @@ -473,7 +472,8 @@ public ModelMap addNewSequenceFileToSample(@PathVariable Long sampleId, @Request } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - IridaFiles.cleanupLocalTemporaryFiles(target, temp); + Files.deleteIfExists(target); + Files.deleteIfExists(temp); } // respond to the client @@ -584,7 +584,8 @@ public ModelMap addNewFast5FileToSample(@PathVariable Long sampleId, @RequestPar } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - IridaFiles.cleanupLocalTemporaryFiles(target, temp); + Files.deleteIfExists(target); + Files.deleteIfExists(temp); } // respond to the client @@ -686,8 +687,10 @@ public ModelMap addNewSequenceFilePairToSample(@PathVariable Long sampleId, } finally { // clean up the temporary files. logger.trace("Deleted temp files"); - IridaFiles.cleanupLocalTemporaryFiles(target1, temp1); - IridaFiles.cleanupLocalTemporaryFiles(target2, temp2); + Files.deleteIfExists(target1); + Files.deleteIfExists(temp1); + Files.deleteIfExists(target2); + Files.deleteIfExists(temp2); } // respond to the client diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java index 790d1591f80..ba26c80d185 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenatorTest.java @@ -5,7 +5,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; @@ -51,8 +50,7 @@ public void testConcatenateFiles() throws IOException, ConcatenateException { SequenceFilePair f1 = new SequenceFilePair(original1, original2); SequenceFilePair f2 = new SequenceFilePair(original3, original4); - IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); - SequenceFilePair concatenateFiles = (SequenceFilePair) iridaConcatenatorTemporaryFile.getSequencingObject(); + SequenceFilePair concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); SequenceFile forward = concatenateFiles.getForwardSequenceFile(); SequenceFile reverse = concatenateFiles.getReverseSequenceFile(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java index 8e4fc949198..5652308314c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SingleEndSequenceFileConcatenatorTest.java @@ -5,7 +5,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaConcatenatorTemporaryFile; import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.Lists; @@ -48,8 +47,7 @@ public void testConcatenateFiles() throws IOException, ConcatenateException { SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); - SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); + SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); SequenceFile newSeqFile = concatenateFiles.getSequenceFile(); @@ -76,8 +74,7 @@ public void testConcatenateFilesZipped() throws IOException, ConcatenateExceptio SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); - SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); + SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); SequenceFile newSeqFile = concatenateFiles.getSequenceFile(); @@ -100,8 +97,7 @@ public void testConcatenateDifferentFileTypes() throws IOException, ConcatenateE SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); - SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); + SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); } @Test(expected = ConcatenateException.class) @@ -114,8 +110,7 @@ public void testConcatenateBadExtension() throws IOException, ConcatenateExcepti SingleEndSequenceFile f1 = new SingleEndSequenceFile(original1); SingleEndSequenceFile f2 = new SingleEndSequenceFile(original2); - IridaConcatenatorTemporaryFile iridaConcatenatorTemporaryFile = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); - SingleEndSequenceFile concatenateFiles = (SingleEndSequenceFile) iridaConcatenatorTemporaryFile.getSequencingObject(); + SingleEndSequenceFile concatenateFiles = concat.concatenateFiles(Lists.newArrayList(f1, f2), newFileName); } private SequenceFile createSequenceFile(String name, String extension) throws IOException { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java index 294cb3ac0b7..a3b3616928e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/processing/impl/unit/GzipFileProcessorTest.java @@ -44,7 +44,7 @@ public class GzipFileProcessorTest { public void setUp() { sequenceFileRepository = mock(SequenceFileRepository.class); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageUtility, Boolean.FALSE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.FALSE, iridaFileStorageUtility); IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); } @@ -67,7 +67,7 @@ public void testExceptionBehaviours() throws IOException { @Test public void testDeleteOriginalFile() throws IOException { - fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageUtility, Boolean.FALSE); + fileProcessor = new GzipFileProcessor(sequenceFileRepository, Boolean.TRUE, iridaFileStorageUtility); final SequenceFile sf = constructSequenceFile(); // compress the file, update the sequence file reference diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 8419f3c14bc..54f380d671e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -19,7 +19,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.samples.SamplesAjaxController; import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; @@ -37,7 +36,6 @@ public class SamplesAjaxControllerTest { private SamplesAjaxController controller; private SequencingObjectService sequencingObjectService; private GenomeAssemblyService genomeAssemblyService; - private IridaFileStorageLocalUtilityImpl iridaFileStorageUtility; /* TEST DATA @@ -61,8 +59,6 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - iridaFileStorageUtility = mock(IridaFileStorageLocalUtilityImpl.class); - IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); From 4d3af6f73a93c490f5c649ca2b6124130711e4af Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 9 Sep 2020 08:11:09 -0500 Subject: [PATCH 139/655] Removed unused import and removed newlines --- .../bioinformatics/irida/processing/impl/GzipFileProcessor.java | 1 + .../irida/service/impl/SequencingObjectServiceImpl.java | 2 +- .../irida/ria/unit/web/samples/SamplesAjaxControllerTest.java | 2 -- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index e526014ea9a..8981f16e287 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -20,6 +20,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; + /** * Handle gzip-ed files (if necessary). This class partially assumes that gzip * compressed files have the extension ".gz" (not for determining whether or not diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java index 5585ef50458..d48eb0dd515 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/SequencingObjectServiceImpl.java @@ -19,7 +19,6 @@ import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.repositories.specification.SampleSequencingObjectSpecification; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; - import com.google.common.collect.ImmutableMap; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; @@ -252,6 +251,7 @@ public SampleSequencingObjectJoin concatenateSequences(List to .getConcatenator(toJoin, iridaFileStorageUtility); SequencingObject concatenated = concatenator.concatenateFiles(toJoin, filename); + SampleSequencingObjectJoin created = createSequencingObjectInSample(concatenated, targetSample); concatenationRepository.save(new SequenceConcatenation(created.getObject(), toJoin)); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index 54f380d671e..9b667c25fad 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -24,7 +24,6 @@ import ca.corefacility.bioinformatics.irida.service.GenomeAssemblyService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.google.common.collect.ImmutableList; @@ -59,7 +58,6 @@ public void setUp() { sequencingObjectService = mock(SequencingObjectService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); MessageSource messageSource = mock(MessageSource.class); - controller = new SamplesAjaxController(sampleService, sequencingObjectService, genomeAssemblyService, messageSource); // Set up mocks From 852f5b9327fe79f60ef841cdec400c1466061dd7 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 9 Sep 2020 09:59:23 -0500 Subject: [PATCH 140/655] Removed unused imports --- .../irida/processing/impl/FastqcFileProcessor.java | 1 - .../ca/corefacility/bioinformatics/irida/util/IridaFiles.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 1360695e164..3b23ade993d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -14,7 +14,6 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 3c985468fa4..319e34051a7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -7,7 +7,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; /** * Static class which has file object operations that require From 117eedc7f06e586f08eed0fca9d520de62ca3c6f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 14 Oct 2020 14:33:45 -0500 Subject: [PATCH 141/655] Added comments. Updated exception thrown instead of a logging statement when trying to clean up the analysis outputs directory. Added prefix to temporary directory for azure files. --- .../irida/model/sequenceFile/Fast5Object.java | 4 +- .../processing/impl/FastqcFileProcessor.java | 37 ++++++++++++------- .../IridaFileStorageAzureUtilityImpl.java | 2 +- .../irida/ria/web/dto/IridaTemporaryFile.java | 6 ++- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java index 2e8fa596162..0957be0c47e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/Fast5Object.java @@ -106,9 +106,7 @@ private Fast5Type setType(SequenceFile file) { String extension = FilenameUtils.getExtension(getFile().getFileName()); // Checks if file is where it should be before it checks if it is gzipped - if (IridaFiles.fileExists(file.getFile()) && file.isGzipped()) { - type = Fast5Object.Fast5Type.ZIPPED; - } else if (extension.equals("gz")) { + if ((IridaFiles.fileExists(file.getFile()) && file.isGzipped()) || extension.equals("gz")) { type = Fast5Object.Fast5Type.ZIPPED; } else if (extension.equals("fast5")) { type = Fast5Object.Fast5Type.SINGLE; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index 3b23ade993d..ad788dffdf4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -96,14 +96,17 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx AnalysisFastQC.AnalysisFastQCBuilder analysis = AnalysisFastQC.builder() .fastqcVersion(FastQCApplication.VERSION) .executionManagerAnalysisId(EXECUTION_MANAGER_ANALYSIS_ID) - .description(messageSource.getMessage("fastqc.file.processor.analysis.description", new Object[] {FastQCApplication.VERSION}, - LocaleContextHolder.getLocale())); + .description(messageSource.getMessage("fastqc.file.processor.analysis.description", + new Object[] { FastQCApplication.VERSION }, LocaleContextHolder.getLocale())); + // Get the local copy if using local storage otherwise it temporarily downloads the file from the object store IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(fileToProcess); - File fastQCSequenceFileToProcess = iridaTemporaryFile.getFile().toFile(); + File fastQCSequenceFileToProcess = iridaTemporaryFile.getFile() + .toFile(); Path outputDirectory = null; try { + // Create temporary directory to hold the analysis output files created by fastqc outputDirectory = Files.createTempDirectory("analysis-output"); try { @@ -115,6 +118,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx OverRepresentedSeqs overRep = new OverRepresentedSeqs(); QCModule[] moduleList = new QCModule[] { basicStats, pbqs, psqs, overRep }; + // If there is a sequence file to process then we run the fastqc modules logger.debug("Launching FastQC analysis modules on all sequences."); while (fastQCSequenceFile.hasNext()) { Sequence sequence = fastQCSequenceFile.next(); @@ -125,6 +129,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx logger.debug("Finished FastQC analysis modules."); + // Create the fastqc images and save to the temporary directory handleBasicStats(basicStats, analysis); handlePerBaseQualityScores(pbqs, analysis, outputDirectory); handlePerSequenceQualityScores(psqs, analysis, outputDirectory); @@ -147,13 +152,16 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx logger.error("Unable to create temporary directory ", e); throw new StorageException("Unable to create temporary directory", e); } finally { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); - try { - // Delete the analysis-output* temp directory - Files.deleteIfExists(outputDirectory); - } catch (IOException e) { - logger.error("Unable to remove directory", e); - } + /* If a temporary file was downloaded from an object store then the file and/or temp directory will be deleted, + * otherwise no action is taken. + */ + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + try { + // Delete the analysis-output* temp directory + Files.deleteIfExists(outputDirectory); + } catch (IOException e) { + throw new StorageException("Unable to delete analysis outputs temp directory [" + e + "]"); + } } } @@ -177,7 +185,8 @@ private void handleBasicStats(BasicStats stats, AnalysisFastQCBuilder analysis) } /** - * Handle writing the {@link PerBaseQualityScores} to the database. + * Handle writing the {@link PerBaseQualityScores} to the database and + * the generated fastqc image to the tempDirectory * * @param scores the {@link PerBaseQualityScores} computed by fastqc. * @param analysis the {@link AnalysisFastQCBuilder} to update. @@ -194,7 +203,8 @@ private void handlePerBaseQualityScores(PerBaseQualityScores scores, AnalysisFas } /** - * Handle writing the {@link PerSequenceQualityScores} to the database. + * Handle writing the {@link PerSequenceQualityScores} to the database and + * the generated fastqc image to the tempDirectory * * @param scores the {@link PerSequenceQualityScores} computed by fastqc. * @param analysis the {@link AnalysisFastQCBuilder} to update. @@ -211,7 +221,8 @@ private void handlePerSequenceQualityScores(PerSequenceQualityScores scores, Ana } /** - * Handle writing the {@link DuplicationLevel} to the database. + * Handle writing the {@link DuplicationLevel} to the database and + * the generated fastqc image to the tempDirectory * * @param duplicationLevel the {@link DuplicationLevel} calculated by fastqc. * @param analysis the {@link AnalysisFastQCBuilder} to update. diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 5562a9800df..db4ae0fa076 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -55,7 +55,7 @@ public IridaTemporaryFile getTemporaryFile(Path file) { try { logger.trace("Getting file from azure [" + file.toString() + "]"); - Path tempDirectory = Files.createTempDirectory(null); + Path tempDirectory = Files.createTempDirectory("azure-tmp-"); Path tempFile = tempDirectory.resolve(file.getFileName().toString()); InputStream initialStream = blobClient.openInputStream(); org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java index 46927df654e..159f8bfd4d7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java @@ -3,7 +3,11 @@ import java.nio.file.Path; /** - * Used as a response for encapsulating a temporary file and it's directory + * Used as a response for encapsulating a temporary file and it's directory. This is + * used to download a file from a cloud object store to a temporary file location + * on the local file system. If this is dto is used then we must remember to cleanup + * the temporary file and/or directory using the cleanupDownloadedLocalTemporaryFiles + * in the iridaFileStorageUtility. */ public class IridaTemporaryFile { From 1dd0672eb3b8f066928f5efd3862d9c5eab3be50 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 14 Oct 2020 18:35:58 -0500 Subject: [PATCH 142/655] Changed package to delete temp analysis outputs directory --- .../irida/processing/impl/FastqcFileProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index ad788dffdf4..d7e64b8ffd3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -15,6 +15,7 @@ import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -158,7 +159,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); try { // Delete the analysis-output* temp directory - Files.deleteIfExists(outputDirectory); + FileUtils.deleteDirectory(outputDirectory.toFile()); } catch (IOException e) { throw new StorageException("Unable to delete analysis outputs temp directory [" + e + "]"); } From d15b2a07ea240feaf6f39b90428d985fca96efa8 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 21 Oct 2020 16:20:42 -0500 Subject: [PATCH 143/655] Moved into filesystem directory --- .../irida/processing/impl/FastqcFileProcessor.java | 2 +- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 1 - .../filesystem/IridaFileStorageLocalUtilityImpl.java | 1 - .../irida/repositories/filesystem/IridaFileStorageUtility.java | 1 - .../web/dto => repositories/filesystem}/IridaTemporaryFile.java | 2 +- 5 files changed, 2 insertions(+), 5 deletions(-) rename src/main/java/ca/corefacility/bioinformatics/irida/{ria/web/dto => repositories/filesystem}/IridaTemporaryFile.java (92%) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index d7e64b8ffd3..a78f0d5c84a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -13,7 +13,7 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.AnalysisOutputFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index db4ae0fa076..d601e213e03 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -19,7 +19,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.azure.storage.blob.BlobClient; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 9e800e0636f..5640d92149c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -20,7 +20,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.util.FileUtils; /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 912af235a39..3b3d206bc8f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -7,7 +7,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.ria.web.dto.IridaTemporaryFile; import com.google.common.collect.Lists; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaTemporaryFile.java similarity index 92% rename from src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java rename to src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaTemporaryFile.java index 159f8bfd4d7..7fbb3a89ce3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/dto/IridaTemporaryFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaTemporaryFile.java @@ -1,4 +1,4 @@ -package ca.corefacility.bioinformatics.irida.ria.web.dto; +package ca.corefacility.bioinformatics.irida.repositories.filesystem; import java.nio.file.Path; From 2dc172a7c93d54c819c83356ed1270526877b050 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 22 Oct 2020 16:49:58 -0500 Subject: [PATCH 144/655] Updated authentication for azure storage utility class --- .../irida/config/services/IridaApiServicesConfig.java | 9 ++++++--- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 5 ++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 9c523a93a03..dfe4c013115 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -135,8 +135,11 @@ public class IridaApiServicesConfig { @Value("${azure.container.name:#{null}}") private String containerName; - @Value("${azure.account.connection.string:#{null}}") - private String connectionStr; + @Value("${azure.container.url:#{null}}") + private String containerUrl; + + @Value("${azure.sas.token:#{null}}") + private String sasToken; @Autowired @@ -307,7 +310,7 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { public IridaFileStorageUtility iridaFileStorageUtility() { IridaFileStorageUtility iridaFileStorageUtility; if(storageType.equalsIgnoreCase("azure")) { - iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(connectionStr, containerName); + iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(containerUrl, sasToken, containerName); } else { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index d601e213e03..edbbcf4c61f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -38,9 +38,8 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobContainerClient containerClient ; @Autowired - public IridaFileStorageAzureUtilityImpl(String connectionStr, String containerName){ - this.blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionStr) - .buildClient(); + public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName){ + this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl).sasToken(sasToken).buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); } From 74049bc35efe3096cf568b220f513eeb6ba1a0ea Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 3 Nov 2020 19:07:08 -0600 Subject: [PATCH 145/655] Updated getFileSize and getFileName methods to not error out if an exception is thrown. --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index edbbcf4c61f..5f7b5502721 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -107,8 +107,9 @@ public String getFileSize(Path file) { } catch (BlobStorageException e) { logger.error("Couldn't calculate size as the file was not found on azure [" + e + "]"); throw new StorageException("Unable to locate file on azure", e); + } finally { + return fileSize; } - return fileSize; } /** @@ -159,9 +160,9 @@ public String getFileName(Path file) { } catch (BlobStorageException e) { logger.error("Couldn't find file on azure [" + e + "]"); throw new StorageException("Unable to locate file on azure", e); + } finally { + return fileName; } - - return fileName; } /** From c43265bd650f0a96e09d9dc086e9775f7dd660c6 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 4 Nov 2020 11:00:19 -0600 Subject: [PATCH 146/655] Removed finally blocks and storageexceptions --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 5f7b5502721..a21f1cbef9e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -106,10 +106,8 @@ public String getFileSize(Path file) { fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties().getBlobSize(), true); } catch (BlobStorageException e) { logger.error("Couldn't calculate size as the file was not found on azure [" + e + "]"); - throw new StorageException("Unable to locate file on azure", e); - } finally { - return fileSize; } + return fileSize; } /** @@ -158,11 +156,9 @@ public String getFileName(Path file) { .split("/"); fileName = blobNameTokens[blobNameTokens.length - 1]; } catch (BlobStorageException e) { - logger.error("Couldn't find file on azure [" + e + "]"); - throw new StorageException("Unable to locate file on azure", e); - } finally { - return fileName; + logger.error("Couldn't retrieve filename. File not found on azure [" + e + "]"); } + return fileName; } /** From 2954fe8656d6f558e5882255d52a951da3639cec Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 4 Nov 2020 15:40:33 -0600 Subject: [PATCH 147/655] Removed unused import --- .../repositories/filesystem/IridaFileStorageAwsUtilityImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index ad3cd4b7cbf..83c1fd263e6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -5,7 +5,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Date; import java.util.List; import java.util.Optional; import java.util.zip.GZIPInputStream; From 5af4797a72ba2d6ae58a2f1b9b6e1941296242ed Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 5 Nov 2020 10:04:42 -0600 Subject: [PATCH 148/655] Updated aws maven package. Changed from using basiccredentials to using the default credentials chain provider (set credentials and region in a credentials and config file) --- pom.xml | 2 +- .../config/services/IridaApiServicesConfig.java | 12 +----------- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 7 ++----- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 5fd1cd19006..ba0bb42978d 100644 --- a/pom.xml +++ b/pom.xml @@ -1206,7 +1206,7 @@ 1.2.0 2.1.0 12.0.0 - 1.11.376 + 1.11.894 4.5.12 diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 8c13d7f66f4..0f72711a6fe 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -135,15 +135,6 @@ public class IridaApiServicesConfig { @Value("${aws.bucket.name:#{null}}") private String awsBucketName; - @Value("${aws.bucket.region:#{null}}") - private String awsBucketRegion; - - @Value("${aws.access.key:#{null}}") - private String awsAccessKey; - - @Value("${aws.secret.key:#{null}}") - private String awsSecretKey; - @Value("${azure.container.name:#{null}}") private String containerName; @@ -322,8 +313,7 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { public IridaFileStorageUtility iridaFileStorageService() { IridaFileStorageUtility iridaFileStorageUtility; if (storageType.equalsIgnoreCase("aws")) { - iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, - awsSecretKey); + iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName); } else if(storageType.equalsIgnoreCase("azure")) { iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(containerUrl, sasToken, containerName); } else { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 83c1fd263e6..bf2708cf1f7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -36,14 +36,11 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsUtilityImpl.class); private String bucketName; - private BasicAWSCredentials awsCreds; private AmazonS3 s3; @Autowired - public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey){ - this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); - this.s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion)) - .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); + public IridaFileStorageAwsUtilityImpl(String bucketName){ + this.s3 = AmazonS3ClientBuilder.standard().build(); this.bucketName = bucketName; } From f58fe1a35dd6f7d642496a54b42dbd6031672e69 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 5 Nov 2020 10:07:43 -0600 Subject: [PATCH 149/655] Undid accidental change to azure maven version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba0bb42978d..d8a72726dca 100644 --- a/pom.xml +++ b/pom.xml @@ -1205,7 +1205,7 @@ 1.3 1.2.0 2.1.0 - 12.0.0 + 12.6.0 1.11.894 4.5.12 From 2901c73843eafa1a559b789725dd241de2df3ade Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 5 Nov 2020 10:09:17 -0600 Subject: [PATCH 150/655] Removed duplicate azure maven package --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index d8a72726dca..eb926c1c42f 100644 --- a/pom.xml +++ b/pom.xml @@ -1205,7 +1205,6 @@ 1.3 1.2.0 2.1.0 - 12.6.0 1.11.894 4.5.12 From a57b40ea72ba5b0615cc388438fbd9641ed78159 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 5 Nov 2020 13:30:06 -0600 Subject: [PATCH 151/655] Added logging statement --- .../irida/processing/impl/FastqcFileProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java index a78f0d5c84a..31607f8bc8b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/FastqcFileProcessor.java @@ -158,6 +158,7 @@ private void processSingleFile(SequenceFile sequenceFile) throws FileProcessorEx */ iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); try { + logger.trace("Removing directory: " + outputDirectory.toString()); // Delete the analysis-output* temp directory FileUtils.deleteDirectory(outputDirectory.toFile()); } catch (IOException e) { From f06ae1cef45206f5b85bd7441f47b2d7429da994 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 6 Nov 2020 10:39:55 -0600 Subject: [PATCH 152/655] Removed unused imports --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index bf2708cf1f7..774ed83b68e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -20,9 +20,6 @@ import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3Object; From 6d893ae30cfc58fda5cc146d9388e5fdba11c0cf Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 6 Nov 2020 12:57:34 -0600 Subject: [PATCH 153/655] Added closing of input stream to s3 object when method getFileInputStream is called --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 774ed83b68e..004485e0866 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -187,14 +187,25 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { + byte[] bytes = new byte[0]; try { S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); logger.trace("Opening input stream to file in s3 bucket [" + file.toString() + "]"); - return s3Object.getObjectContent(); + try { + /* + * We get the bytes of the content and store it in a byte array + * so we can close the inputstream from s3 before returning + */ + bytes = s3Object.getObjectContent().readAllBytes(); + s3Object.close(); + } catch (IOException e) { + logger.error("Unable to close connection to s3object: " + e); + } } catch (AmazonServiceException e) { logger.error("Couldn't read file from s3 bucket [" + e + "]"); throw new StorageException("Unable to locate file in s3 bucket", e); } + return new ByteArrayInputStream(bytes); } /** From 206263d4fbd866238220742e7646a8fd816e6cd5 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 17 Nov 2020 16:07:05 -0600 Subject: [PATCH 154/655] Updated aws storage utility to use config values from irida conf file and updated to use try/finally to make sure a connection to the s3object is closed in case of success or failure --- .../services/IridaApiServicesConfig.java | 12 +++- .../IridaFileStorageAwsUtilityImpl.java | 68 ++++++++----------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 0f72711a6fe..8c13d7f66f4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -135,6 +135,15 @@ public class IridaApiServicesConfig { @Value("${aws.bucket.name:#{null}}") private String awsBucketName; + @Value("${aws.bucket.region:#{null}}") + private String awsBucketRegion; + + @Value("${aws.access.key:#{null}}") + private String awsAccessKey; + + @Value("${aws.secret.key:#{null}}") + private String awsSecretKey; + @Value("${azure.container.name:#{null}}") private String containerName; @@ -313,7 +322,8 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { public IridaFileStorageUtility iridaFileStorageService() { IridaFileStorageUtility iridaFileStorageUtility; if (storageType.equalsIgnoreCase("aws")) { - iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName); + iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, + awsSecretKey); } else if(storageType.equalsIgnoreCase("azure")) { iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(containerUrl, sasToken, containerName); } else { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 004485e0866..83a6bb9ebfc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -20,6 +20,9 @@ import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3Object; @@ -33,11 +36,14 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility{ private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsUtilityImpl.class); private String bucketName; + private BasicAWSCredentials awsCreds; private AmazonS3 s3; @Autowired - public IridaFileStorageAwsUtilityImpl(String bucketName){ - this.s3 = AmazonS3ClientBuilder.standard().build(); + public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey){ + this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); + this.s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion)) + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); this.bucketName = bucketName; } @@ -51,19 +57,19 @@ public IridaTemporaryFile getTemporaryFile(Path file) { Path tempDirectory = Files.createTempDirectory("aws-tmp-"); Path tempFile = tempDirectory.resolve(file.getFileName().toString()); - S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); - S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); - - org.apache.commons.io.FileUtils.copyInputStreamToFile(s3ObjectInputStream, tempFile.toFile()); - - s3ObjectInputStream.close(); - s3Object.close(); + try(S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { + S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); + org.apache.commons.io.FileUtils.copyInputStreamToFile(s3ObjectInputStream, tempFile.toFile()); + s3ObjectInputStream.close(); + } catch (AmazonServiceException e) { + logger.error(e.getErrorMessage()); + throw new StorageException(e.getMessage()); + } catch (Exception e) { + logger.error("Unable to get object from aws S3: " + e); + throw new StorageException(e.getMessage()); + } return new IridaTemporaryFile(tempFile, tempDirectory); - - } catch (AmazonServiceException e) { - logger.error(e.getErrorMessage()); - throw new StorageException(e.getMessage()); } catch (FileNotFoundException e) { logger.error(e.getMessage()); throw new StorageException(e.getMessage()); @@ -105,19 +111,14 @@ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTempora @Override public String getFileSize(Path file) { String fileSize = "N/A"; - try { - S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { fileSize = FileUtils.humanReadableByteCount(s3Object.getObjectMetadata() .getContentLength(), true); - try { - s3Object.close(); - } catch (IOException e) { - logger.error("Unable to close connection to s3object: " + e); - } } catch (AmazonServiceException e) { logger.error("Unable to get file size from s3 bucket: " + e); + } catch (IOException e) { + logger.error(e.getMessage()); } - return fileSize; } @@ -157,18 +158,16 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - try { - S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + try(S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { // Since the file system is virtual the full file path is the file name. // We split it on "/" and get the last token which is the actual file name. String[] nameTokens = s3Object.getKey() .split("/"); fileName = nameTokens[nameTokens.length - 1]; - s3Object.close(); } catch (AmazonServiceException e) { logger.error("Couldn't find file [" + e + "]"); - } catch (IOException e) { - logger.error("Unable to close connection to s3object: " + e); + } catch (Exception e) { + logger.error(e.getMessage()); } return fileName; @@ -188,22 +187,13 @@ public boolean fileExists(Path file) { @Override public InputStream getFileInputStream(Path file) { byte[] bytes = new byte[0]; - try { - S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); - logger.trace("Opening input stream to file in s3 bucket [" + file.toString() + "]"); - try { - /* - * We get the bytes of the content and store it in a byte array - * so we can close the inputstream from s3 before returning - */ - bytes = s3Object.getObjectContent().readAllBytes(); - s3Object.close(); - } catch (IOException e) { - logger.error("Unable to close connection to s3object: " + e); - } + try(S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { + bytes = s3Object.getObjectContent().readAllBytes(); } catch (AmazonServiceException e) { logger.error("Couldn't read file from s3 bucket [" + e + "]"); throw new StorageException("Unable to locate file in s3 bucket", e); + } catch (Exception e) { + logger.error(e.getMessage()); } return new ByteArrayInputStream(bytes); } From 2ff8d49c203b54e1df18e5388736b81378b1323f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 18 Nov 2020 15:10:38 -0600 Subject: [PATCH 155/655] Added cause of storageexceptions and added s3ObjectInputStream to try-with-resources block --- .../IridaFileStorageAwsUtilityImpl.java | 75 ++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 83a6bb9ebfc..5c85eab86a3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -32,7 +32,7 @@ * Component implementation of file utitlities for aws storage */ @Component -public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility{ +public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAwsUtilityImpl.class); private String bucketName; @@ -40,10 +40,12 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility{ private AmazonS3 s3; @Autowired - public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey){ + public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); - this.s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion)) - .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); + this.s3 = AmazonS3ClientBuilder.standard() + .withRegion(Regions.fromName(bucketRegion)) + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .build(); this.bucketName = bucketName; } @@ -55,27 +57,24 @@ public IridaTemporaryFile getTemporaryFile(Path file) { try { logger.trace("Getting file from aws s3 [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory("aws-tmp-"); - Path tempFile = tempDirectory.resolve(file.getFileName().toString()); + Path tempFile = tempDirectory.resolve(file.getFileName() + .toString()); - try(S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { - S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); + try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { org.apache.commons.io.FileUtils.copyInputStreamToFile(s3ObjectInputStream, tempFile.toFile()); - s3ObjectInputStream.close(); } catch (AmazonServiceException e) { - logger.error(e.getErrorMessage()); - throw new StorageException(e.getMessage()); - } catch (Exception e) { - logger.error("Unable to get object from aws S3: " + e); - throw new StorageException(e.getMessage()); + logger.error(e.getMessage()); + throw new StorageException("Unable to read object from aws s3 bucket", e); } return new IridaTemporaryFile(tempFile, tempDirectory); } catch (FileNotFoundException e) { logger.error(e.getMessage()); - throw new StorageException(e.getMessage()); + throw new StorageException("Unable to resolve temp file in temp directory", e); } catch (IOException e) { logger.error(e.getMessage()); - throw new StorageException(e.getMessage()); + throw new StorageException("Unable to create temp directory", e); } } @@ -85,23 +84,28 @@ public IridaTemporaryFile getTemporaryFile(Path file) { @Override public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTemporaryFile) { try { - if(iridaTemporaryFile.getFile() != null && Files.isRegularFile(iridaTemporaryFile.getFile())) { - logger.trace("Cleaning up temporary file downloaded from aws s3 [" + iridaTemporaryFile.getFile().toString() + "]"); + if (iridaTemporaryFile.getFile() != null && Files.isRegularFile(iridaTemporaryFile.getFile())) { + logger.trace("Cleaning up temporary file downloaded from aws s3 [" + iridaTemporaryFile.getFile() + .toString() + "]"); Files.delete(iridaTemporaryFile.getFile()); } } catch (IOException e) { logger.error("Unable to delete local file", e); - throw new StorageException(e.getMessage()); + throw new StorageException("Unable to delete local file", e); } try { - if(iridaTemporaryFile.getDirectoryPath() != null && Files.isDirectory(iridaTemporaryFile.getDirectoryPath())) { - logger.trace("Cleaning up temporary directory created for aws s3 temporary file [" + iridaTemporaryFile.getDirectoryPath().toString() + "]"); - org.apache.commons.io.FileUtils.deleteDirectory(iridaTemporaryFile.getDirectoryPath().toFile()); + if (iridaTemporaryFile.getDirectoryPath() != null && Files.isDirectory( + iridaTemporaryFile.getDirectoryPath())) { + logger.trace("Cleaning up temporary directory created for aws s3 temporary file [" + + iridaTemporaryFile.getDirectoryPath() + .toString() + "]"); + org.apache.commons.io.FileUtils.deleteDirectory(iridaTemporaryFile.getDirectoryPath() + .toFile()); } } catch (IOException e) { logger.error("Unable to delete local directory", e); - throw new StorageException(e.getMessage()); + throw new StorageException("Unable to delete local directory", e); } } @@ -130,7 +134,8 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque try { logger.trace("Uploading file to s3 bucket: [" + target.getFileName() + "]"); s3.putObject(bucketName, getAwsFileAbsolutePath(target), source.toFile()); - logger.trace("File uploaded to s3 bucket: [" + s3.getUrl(bucketName, target.toAbsolutePath().toString()) + "]"); + logger.trace("File uploaded to s3 bucket: [" + s3.getUrl(bucketName, target.toAbsolutePath() + .toString()) + "]"); } catch (AmazonServiceException e) { logger.error("Unable to upload file to s3 bucket: " + e); throw new StorageException("Unable to upload file s3 bucket", e); @@ -149,7 +154,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque * {@inheritDoc} */ @Override - public boolean storageTypeIsLocal(){ + public boolean storageTypeIsLocal() { return false; } @@ -158,7 +163,7 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - try(S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { + try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { // Since the file system is virtual the full file path is the file name. // We split it on "/" and get the last token which is the actual file name. String[] nameTokens = s3Object.getKey() @@ -187,8 +192,9 @@ public boolean fileExists(Path file) { @Override public InputStream getFileInputStream(Path file) { byte[] bytes = new byte[0]; - try(S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { - bytes = s3Object.getObjectContent().readAllBytes(); + try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { + bytes = s3Object.getObjectContent() + .readAllBytes(); } catch (AmazonServiceException e) { logger.error("Couldn't read file from s3 bucket [" + e + "]"); throw new StorageException("Unable to locate file in s3 bucket", e); @@ -206,8 +212,8 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream inputStream = getFileInputStream(file)) { byte[] bytes = new byte[2]; inputStream.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) - && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC + >> 8))); } } @@ -219,8 +225,9 @@ public boolean isGzipped(Path file) throws IOException { * @return */ private String getAwsFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - if(absolutePath.charAt(0) == '/') { + String absolutePath = file.toAbsolutePath() + .toString(); + if (absolutePath.charAt(0) == '/') { absolutePath = file.toAbsolutePath() .toString() .substring(1); @@ -236,7 +243,8 @@ public void appendToFile(Path target, SequenceFile file) throws IOException { IridaTemporaryFile iridaTemporaryFile = getTemporaryFile(file.getFile()); try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(iridaTemporaryFile.getFile().toFile()).getChannel()) { + try (FileChannel in = new FileInputStream(iridaTemporaryFile.getFile() + .toFile()).getChannel()) { for (long p = 0, l = in.size(); p < l; ) { p += in.transferTo(p, l - p, out); } @@ -276,8 +284,7 @@ public String getFileExtension(List sequencingObject selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { throw new IOException( - "Extensions of files do not match " + currentExtension + " vs " - + selectedExtension); + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } } From 02f2e31b46695cba575b365b780229181fccea40 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 19 Nov 2020 10:19:03 -0600 Subject: [PATCH 156/655] Updated directory check at startup not to crash if using basedirectories that don't exist on local storage but are set for cloud storage --- .../repository/IridaApiFilesystemRepositoryConfig.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java index 8d976ea9e52..af900b28503 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java @@ -39,6 +39,9 @@ public class IridaApiFilesystemRepositoryConfig { private @Value("${assembly.file.base.directory}") String assemblyFileBaseDirectory; + private @Value("${irida.storage.type}") + String storageType; + @Autowired private ApplicationContext applicationContext; @@ -88,9 +91,11 @@ public Path assemblyFileBaseDirectory() throws IOException { return getExistingPathOrThrow(assemblyFileBaseDirectory); } + //FIXME Update the code to check if the connection to a cloud provider + // is successful or not as well as check if path is writeable for local file storage private Path getExistingPathOrThrow(String directory) { Path baseDirectory = Paths.get(directory); - if (!Files.exists(baseDirectory)) { + if (!Files.exists(baseDirectory) && storageType.equals("local")) { throw new IllegalStateException( String.format("Cannot continue startup; base directory [%s] does not exist!", baseDirectory.toString())); From 65c1e26c2e09e34aa94e8da142691fbdb774731e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 19 Nov 2020 17:28:33 -0600 Subject: [PATCH 157/655] Updated reference file page to display upload progress of files --- .../js/apis/projects/reference-files.js | 20 ++++++ .../reference-files/ReferenceFiles.jsx | 65 ++++++++++++++----- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/src/main/webapp/resources/js/apis/projects/reference-files.js b/src/main/webapp/resources/js/apis/projects/reference-files.js index 66fb0bdeeea..15ce9251246 100644 --- a/src/main/webapp/resources/js/apis/projects/reference-files.js +++ b/src/main/webapp/resources/js/apis/projects/reference-files.js @@ -41,3 +41,23 @@ export async function removeProjectReferenceFile(projectId, fileId) { throw new Error(error.response.data.error); }); } + +/** + * Remove the reference file from the project id provided + * @param {projectId} project for which to remove reference file from + * @param {formData} List of files to upload + * @param config Configuration options for upload + * @return {Promise<*>} `data` contains the OK response; `error` contains error information if an error occurred. + */ +export async function uploadProjectReferenceFiles(projectId, formData, config) { + return await axios + .post( + setBaseUrl(`ajax/reference-files/project/${projectId}`), + formData, + config + ) + .then(({ data }) => data) + .catch((error) => { + throw new Error(error.response.data.error); + }); +} diff --git a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx index 4f28f1afc0e..44ad7fbb627 100644 --- a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx +++ b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx @@ -1,12 +1,14 @@ import React from "react"; -import { notification, Space, Table, Typography } from "antd"; +import { notification, Progress, Space, Table, Typography } from "antd"; import { InfoAlert } from "../../../components/alerts"; -import { setBaseUrl } from "../../../utilities/url-utilities"; import { downloadProjectReferenceFile, getProjectReferenceFiles, removeProjectReferenceFile, + uploadProjectReferenceFiles, } from "../../../apis/projects/reference-files"; +import { getProjectInfo } from "../../../apis/projects/projects"; + import { formatInternationalizedDateTime } from "../../../utilities/date-utilities"; import { ContentLoading } from "../../../components/loader"; import { @@ -15,8 +17,6 @@ import { } from "../../../components/Buttons"; import { DragUpload } from "../../../components/files/DragUpload"; -import { getProjectInfo } from "../../../apis/projects/projects"; - const { Title } = Typography; /** @@ -29,6 +29,8 @@ export function ReferenceFiles() { const [projectInfo, setProjectInfo] = React.useState(null); const [loading, setLoading] = React.useState(true); + const [progress, setProgress] = React.useState(0); + const pathRegx = new RegExp(/projects\/(\d+)/); const projectId = window.location.pathname.match(pathRegx)[1]; @@ -133,27 +135,52 @@ export function ReferenceFiles() { }); } - // Options for the Ant Design upload component - const referenceFileUploadOptions = { - multiple: true, - accept: ".fasta", - showUploadList: false, - action: setBaseUrl(`ajax/reference-files/project/${projectId}`), - onChange(info) { - const { status } = info.file; - if (status === "done") { + // Custom request for drag upload to display progress of uploads + const uploadFiles = async (options) => { + const { onSuccess, onError, file, onProgress } = options; + + const formData = new FormData(); + const config = { + headers: { "content-type": "multipart/form-data" }, + onUploadProgress: (event) => { + const percent = Math.floor((event.loaded / event.total) * 100); + setProgress(percent); + if (percent === 100) { + setTimeout(() => setProgress(0), 1000); + } + onProgress({ percent: (event.loaded / event.total) * 100 }); + }, + }; + formData.append("file", file); + + uploadProjectReferenceFiles(projectId, formData, config) + .then(() => { + onSuccess("Ok"); notification.success({ message: `${i18n( "ReferenceFile.uploadFileSuccess", - info.file.name, + file.name, projectInfo.projectName )}`, }); updateReferenceFileTable(); - } else if (status === "error") { - notification.error({ message: info.file.response.error }); - } - }, + }) + .catch((error) => { + onError("Error"); + notification.error({ + message: + "Upload failed for reference file " + + file.name + + ". " + + error.message, + }); + }); + }; + + // Options for the Ant Design upload component + const referenceFileUploadOptions = { + multiple: true, + accept: ".fasta", }; return ( @@ -163,6 +190,8 @@ export function ReferenceFiles() { {projectInfo && projectInfo.canManage ? ( } + customRequest={uploadFiles} uploadText={i18n("ReferenceFile.clickorDrag")} uploadHint={uploadHintMessage[referenceFileUploadOptions.multiple]} /> From 9bc0989245f31a198d5459432812e82774d55b19 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 19 Nov 2020 17:38:33 -0600 Subject: [PATCH 158/655] Updated failing tests --- .../irida/ria/unit/web/files/SequenceFileControllerTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java index c29db2578cb..f0114331766 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/SequenceFileControllerTest.java @@ -24,7 +24,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.web.files.SequenceFileController; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; - +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Unit Tests for @{link SequenceFileController} @@ -36,6 +36,7 @@ public class SequenceFileControllerTest { public static final String FILE_PATH = "src/test/resources/files/test_file.fastq"; private static final Logger logger = LoggerFactory.getLogger(SequenceFileControllerTest.class); private SequenceFileController controller; + private IridaFileStorageUtility iridaFileStorageUtility; // Services private SequencingObjectService objectService; @@ -44,6 +45,8 @@ public class SequenceFileControllerTest { public void setUp() { objectService = mock(SequencingObjectService.class); controller = new SequenceFileController(objectService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); Path path = Paths.get(FILE_PATH); SequenceFile file = new SequenceFile(path); From bec7d1e389352ee79f47fba377d91e9673fd70b0 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 23 Nov 2020 15:36:50 -0600 Subject: [PATCH 159/655] Updated azure storage file utility to use try with resources to close stream. Updated aws file utility to use try with resources to close stream when reading bytes --- .../workflow/analysis/AnalysisFastQC.java | 10 +- .../IridaFileStorageAwsUtilityImpl.java | 20 ++-- .../IridaFileStorageAzureUtilityImpl.java | 98 +++++++++++-------- 3 files changed, 62 insertions(+), 66 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java index 97742155f96..b811e34937f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisFastQC.java @@ -2,7 +2,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.OverrepresentedSequence; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableSet; @@ -395,13 +394,6 @@ public Set getOverrepresentedSequences() { */ private byte[] getBytesForFile(String key) { AnalysisOutputFile chart = getAnalysisOutputFile(key); - byte[] bytes = new byte[0]; - try { - bytes = IridaFiles.getBytesForFile(chart.getFile()); - } catch (IOException e){ - logger.error("Unable to read fastqc file.", e); - } finally { - return bytes; - } + return chart.getBytesForFile(); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index c7facecc745..ddac504e03e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -28,7 +28,6 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; - /** * Component implementation of file utitlities for aws storage */ @@ -299,23 +298,16 @@ public String getFileExtension(List sequencingObject */ @Override public byte[] readAllBytes(Path file) { - byte [] bytes = new byte[0]; - try { - S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); - S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent(); + byte[] bytes = new byte[0]; + try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { bytes = s3ObjectInputStream.readAllBytes(); - - try { - s3Object.close(); - } catch (IOException e) { - logger.error("Unable to close connection to s3object: " + e); - } + } catch (AmazonServiceException e) { + logger.error(e.getMessage()); + throw new StorageException("Unable to read object from aws s3 bucket", e); } catch (IOException e) { logger.error("Couldn't get bytes from file [" + e + "]"); - } catch (AmazonServiceException e) { - logger.error("Couldn't read file from aws s3 bucket [" + e + "]"); } - return bytes; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 364c0508e03..e9adf859297 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; +import java.io.ByteArrayInputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -36,11 +37,13 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageAzureUtilityImpl.class); private BlobServiceClient blobServiceClient; - private BlobContainerClient containerClient ; + private BlobContainerClient containerClient; @Autowired - public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName){ - this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl).sasToken(sasToken).buildClient(); + public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { + this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl) + .sasToken(sasToken) + .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); } @@ -49,23 +52,23 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St */ @Override public IridaTemporaryFile getTemporaryFile(Path file) { - // We set the blobClient "path" to which file we want to get - BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - try { - logger.trace("Getting file from azure [" + file.toString() + "]"); - Path tempDirectory = Files.createTempDirectory("azure-tmp-"); - Path tempFile = tempDirectory.resolve(file.getFileName().toString()); - InputStream initialStream = blobClient.openInputStream(); - org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); - initialStream.close(); - return new IridaTemporaryFile(tempFile, tempDirectory); + // We set the blobClient "path" to which file we want to get + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + try (InputStream initialStream = blobClient.openInputStream()) { + logger.trace("Getting file from azure [" + file.toString() + "]"); + Path tempDirectory = Files.createTempDirectory("azure-tmp-"); + Path tempFile = tempDirectory.resolve(file.getFileName() + .toString()); + org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); + return new IridaTemporaryFile(tempFile, tempDirectory); + } catch (IOException e) { + logger.error(e.getMessage()); + throw new StorageException(e.getMessage()); + } } catch (BlobStorageException e) { logger.error("Couldn't find file on azure [" + e + "]"); throw new StorageException("Unable to locate file on azure", e); - } catch (IOException e) { - logger.error(e.getMessage()); - throw new StorageException(e.getMessage()); } } @@ -75,8 +78,9 @@ public IridaTemporaryFile getTemporaryFile(Path file) { @Override public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTemporaryFile) { try { - if(iridaTemporaryFile.getFile() != null && Files.isRegularFile(iridaTemporaryFile.getFile())) { - logger.trace("Cleaning up temporary file downloaded from azure [" + iridaTemporaryFile.getFile().toString() + "]"); + if (iridaTemporaryFile.getFile() != null && Files.isRegularFile(iridaTemporaryFile.getFile())) { + logger.trace("Cleaning up temporary file downloaded from azure [" + iridaTemporaryFile.getFile() + .toString() + "]"); Files.delete(iridaTemporaryFile.getFile()); } } catch (IOException e) { @@ -85,9 +89,13 @@ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTempora } try { - if(iridaTemporaryFile.getDirectoryPath() != null && Files.isDirectory(iridaTemporaryFile.getDirectoryPath())) { - logger.trace("Cleaning up temporary directory created for azure temporary file [" + iridaTemporaryFile.getDirectoryPath().toString() + "]"); - org.apache.commons.io.FileUtils.deleteDirectory(iridaTemporaryFile.getDirectoryPath().toFile()); + if (iridaTemporaryFile.getDirectoryPath() != null && Files.isDirectory( + iridaTemporaryFile.getDirectoryPath())) { + logger.trace("Cleaning up temporary directory created for azure temporary file [" + + iridaTemporaryFile.getDirectoryPath() + .toString() + "]"); + org.apache.commons.io.FileUtils.deleteDirectory(iridaTemporaryFile.getDirectoryPath() + .toFile()); } } catch (IOException e) { logger.error("Unable to delete local directory", e); @@ -104,7 +112,8 @@ public String getFileSize(Path file) { try { // We set the blobClient "path" to which we want to get a file size for BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties().getBlobSize(), true); + fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties() + .getBlobSize(), true); } catch (BlobStorageException e) { logger.error("Couldn't calculate size as the file was not found on azure [" + e + "]"); } @@ -141,7 +150,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque * {@inheritDoc} */ @Override - public boolean storageTypeIsLocal(){ + public boolean storageTypeIsLocal() { return false; } @@ -169,7 +178,7 @@ public String getFileName(Path file) { @Override public boolean fileExists(Path file) { BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - if(blobClient.exists()) { + if (blobClient.exists()) { return true; } return false; @@ -180,15 +189,20 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { - BlobClient blobClient; + byte[] bytes = new byte[0]; try { logger.trace("Opening input stream to file on azure [" + file.toString() + "]"); - blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - return blobClient.openInputStream(); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + try (BlobInputStream blobInputStream = blobClient.openInputStream()) { + bytes = blobInputStream.readAllBytes(); + } catch (IOException e) { + logger.error("Couldn't get bytes from file [" + e + "]"); + } } catch (BlobStorageException e) { logger.error("Couldn't read file from azure [" + e + "]"); throw new StorageException("Unable to locate file on azure", e); } + return new ByteArrayInputStream(bytes); } /** @@ -199,8 +213,8 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { byte[] bytes = new byte[2]; is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) - && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC + >> 8))); } } @@ -212,7 +226,8 @@ public void appendToFile(Path target, SequenceFile file) throws IOException { IridaTemporaryFile iridaTemporaryFile = getTemporaryFile(file.getFile()); try (FileChannel out = FileChannel.open(target, StandardOpenOption.CREATE, StandardOpenOption.APPEND, StandardOpenOption.WRITE)) { - try (FileChannel in = new FileInputStream(iridaTemporaryFile.getFile().toFile()).getChannel()) { + try (FileChannel in = new FileInputStream(iridaTemporaryFile.getFile() + .toFile()).getChannel()) { for (long p = 0, l = in.size(); p < l; ) { p += in.transferTo(p, l - p, out); } @@ -252,8 +267,7 @@ public String getFileExtension(List sequencingObject selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { throw new IOException( - "Extensions of files do not match " + currentExtension + " vs " - + selectedExtension); + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } } @@ -266,21 +280,18 @@ public String getFileExtension(List sequencingObject */ @Override public byte[] readAllBytes(Path file) { - BlobInputStream blobInputStream = null; BlobClient blobClient; - byte [] bytes = new byte[0]; + byte[] bytes = new byte[0]; try { blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - blobInputStream = blobClient.openInputStream(); - bytes = blobInputStream.readAllBytes(); - } catch (IOException e) { - logger.error("Couldn't get bytes from file [" + e + "]"); + try (BlobInputStream blobInputStream = blobClient.openInputStream()) { + bytes = blobInputStream.readAllBytes(); + } catch (IOException e) { + logger.error("Couldn't get bytes from file [" + e + "]"); + } } catch (BlobStorageException e) { logger.error("Couldn't read file from azure [" + e + "]"); } - if(blobInputStream != null) { - blobInputStream.close(); - } return bytes; } @@ -292,8 +303,9 @@ public byte[] readAllBytes(Path file) { * @return */ private String getAzureFileAbsolutePath(Path file) { - String absolutePath = file.toAbsolutePath().toString(); - if(absolutePath.charAt(0) == '/') { + String absolutePath = file.toAbsolutePath() + .toString(); + if (absolutePath.charAt(0) == '/') { absolutePath = file.toAbsolutePath() .toString() .substring(1); From ed20220c5c0796a2b53b68cb6a55bc69daf3fb20 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Nov 2020 13:06:30 -0600 Subject: [PATCH 160/655] Added contentloading spinner when retrieving data for outputfiles --- src/main/resources/i18n/messages.properties | 1 + .../components/AnalysisTabularPreview.jsx | 8 +++++-- .../components/AnalysisTextPreview.jsx | 22 ++++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index a08e4492a81..ad480f3d3eb 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2191,6 +2191,7 @@ AnalysisOutputs.downloadAllFiles=Download All Files AnalysisOutputs.noOutputsAvailable=No outputs available to display AnalysisOutputs.excelParsingError=There was an error processing a preview for this excel file. Please download to view the output. AnalysisOutputs.unableToReadImageFile=Unable to read the image file. Please download to view the image. +AnalysisOutputs.retrievingFileData=Retrieving file data # ========================================================================================== # # ANALYSIS PHYLOGENETIC TREE COMPONENT # diff --git a/src/main/webapp/resources/js/pages/analysis/components/AnalysisTabularPreview.jsx b/src/main/webapp/resources/js/pages/analysis/components/AnalysisTabularPreview.jsx index ca2c9d74389..3dae0e14771 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/AnalysisTabularPreview.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/AnalysisTabularPreview.jsx @@ -23,6 +23,7 @@ export function AnalysisTabularPreview({ output }) { const [fileRows, setFileRows] = useState([]); const [fileCols, setFileCols] = useState([]); const MAX_TABLE_ROWS_PER_PAGE = 5; + const [loading, setLoading] = useState(false); /* * Get tabular file output data from the start of a file to the end @@ -31,14 +32,16 @@ export function AnalysisTabularPreview({ output }) { * can be changed as such */ useEffect(() => { + setLoading(true); getDataViaLines({ start: 0, end: null, submissionId: output.analysisSubmissionId, - fileId: output.id - }).then(data => { + fileId: output.id, + }).then((data) => { setFileRows(parseRows(data.lines, data.start, fileExtCSV)); setFileCols(parseHeader(firstLine, fileExtCSV)); + setLoading(false); }); }, []); @@ -49,6 +52,7 @@ export function AnalysisTabularPreview({ output }) { { + setLoading(true); getDataViaChunks({ submissionId: output.analysisSubmissionId, fileId: output.id, seek: 0, - chunk: getNewChunkSize(0, output.fileSizeBytes, chunkSize) - }).then(data => { + chunk: getNewChunkSize(0, output.fileSizeBytes, chunkSize), + }).then((data) => { setSavedText(data.text); setFilePointer(data.filePointer); setFileRows(data.text); document.getElementById( `${output.filename}-preview-status` ).innerText = fileSizeLoaded(data.filePointer, output.fileSizeBytes); + setLoading(false); }); }, []); @@ -66,12 +69,13 @@ export default function AnalysisTextPreview({ output }) { scollElement.scrollHeight && getNewChunkSize(filePointer, output.fileSizeBytes, chunkSize) >= 0 ) { + setLoading(true); getDataViaChunks({ submissionId: output.analysisSubmissionId, fileId: output.id, seek: filePointer, - chunk: getNewChunkSize(filePointer, output.fileSizeBytes, chunkSize) - }).then(data => { + chunk: getNewChunkSize(filePointer, output.fileSizeBytes, chunkSize), + }).then((data) => { if (data.text !== null) { setSavedText(savedText + data.text); setFilePointer(data.filePointer); @@ -80,6 +84,7 @@ export default function AnalysisTextPreview({ output }) { `${output.filename}-preview-status` ).innerText = fileSizeLoaded(data.filePointer, output.fileSizeBytes); } + setLoading(false); }); } } @@ -101,6 +106,13 @@ export default function AnalysisTextPreview({ output }) { {fileRows}
    +
    + {loading ? ( + + ) : null} +
    ); From a88665daf66a39c763ba3f98fc9a785f7b6f7da3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Nov 2020 16:41:48 -0600 Subject: [PATCH 161/655] Fixed undefined text for file name --- .../irida/model/workflow/analysis/AnalysisOutputFile.java | 3 +-- .../js/pages/projects/reference-files/ReferenceFiles.jsx | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 08cc7f5d2c8..a74c69b5608 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -196,9 +196,8 @@ public byte[] getBytesForFile() { bytes = IridaFiles.getBytesForFile(getFile()); } catch (IOException e) { logger.error("Unable to read file.", e); - } finally { - return bytes; } + return bytes; } /** diff --git a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx index 44ad7fbb627..1ea46d5e629 100644 --- a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx +++ b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx @@ -92,7 +92,7 @@ export function ReferenceFiles() { notification.success({ message: i18n( "ReferenceFile.downloadingFileSuccess", - file.label, + file.name, projectInfo.projectName ), }); From acd47bd853a94535ff115f45c35aeb39b49066fe Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Nov 2020 16:51:56 -0600 Subject: [PATCH 162/655] Added translation --- src/main/resources/i18n/messages.properties | 1 + .../pages/projects/reference-files/ReferenceFiles.jsx | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index a08e4492a81..702d6722eb4 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2329,6 +2329,7 @@ ReferenceFile.removeTooltip=Remove reference file from project ReferenceFile.confirmText=Remove reference file {0} from project {1}? ReferenceFile.clickorDrag=Click or drag file to this area to upload ReferenceFile.singleOrMultiple=Supports single or multiple reference file uploads. +ReferenceFile.uploadFileError=Upload failed for reference file {0}. {1} server.projects.reference-file.delete-success=Reference file {0} has been removed from project {1} server.projects.reference-file.delete-error=There was an error removing reference file {0} from project {1} diff --git a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx index 1ea46d5e629..c4e298f95de 100644 --- a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx +++ b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx @@ -168,11 +168,11 @@ export function ReferenceFiles() { .catch((error) => { onError("Error"); notification.error({ - message: - "Upload failed for reference file " + - file.name + - ". " + - error.message, + message: i18n( + "ReferenceFile.uploadFileError", + file.name, + error.message + ), }); }); }; From 084485fd4e8257e3efa0d431e86ae1f925450db7 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 26 Nov 2020 13:54:23 -0600 Subject: [PATCH 163/655] Removed reference file UI changes and made a separate PR into dev --- .../web/ajax/ReferenceFileAjaxController.java | 5 +- .../UIProjectReferenceFileService.java | 14 +--- src/main/resources/i18n/messages.properties | 2 - .../js/apis/projects/reference-files.js | 20 ------ .../reference-files/ReferenceFiles.jsx | 64 ++++++------------- .../ReferenceFileAjaxControllerTest.java | 2 +- .../UIProjectReferenceFileServiceTest.java | 2 +- 7 files changed, 24 insertions(+), 85 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/ReferenceFileAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/ReferenceFileAjaxController.java index 9abda72715d..67a971d7e66 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/ReferenceFileAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/ReferenceFileAjaxController.java @@ -97,12 +97,11 @@ public void downloadReferenceFile(@PathVariable Long fileId, HttpServletResponse * Get the reference files for a project * * @param projectId the ID of the project - * @param locale locale of the logged in user * @return information about the reference files in the project */ @GetMapping("/{projectId}") - public ResponseEntity> getReferenceFilesForProject(@PathVariable Long projectId, Locale locale) { - return ResponseEntity.ok(uiProjectReferenceFileService.getReferenceFilesForProject(projectId, locale)); + public ResponseEntity> getReferenceFilesForProject(@PathVariable Long projectId) { + return ResponseEntity.ok(uiProjectReferenceFileService.getReferenceFilesForProject(projectId)); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java index ce312052abc..158591f96a6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java @@ -21,7 +21,6 @@ import ca.corefacility.bioinformatics.irida.model.joins.Join; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; -import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.references.UIReferenceFile; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; @@ -129,24 +128,15 @@ public void downloadReferenceFile(Long fileId, HttpServletResponse response) thr * Get the reference files for a project * * @param projectId the ID of the project - * @param locale locale of the logged in user * @return information about the reference files in the project */ - public List getReferenceFilesForProject(Long projectId, Locale locale) { + public List getReferenceFilesForProject(Long projectId) { Project project = projectService.read(projectId); // Let's get the reference files List> joinList = referenceFileService.getReferenceFilesForProject(project); List refFiles = new ArrayList<>(); for (Join join : joinList) { - try { - refFiles.add(new UIReferenceFile(join, FileUtilities.humanReadableByteCount(Files.size(join.getObject().getFile()), true))); - } catch (IOException e) { - logger.error("Cannot find the size of file " + join.getObject() - .getLabel()); - UIReferenceFile uiReferenceFile = new UIReferenceFile(join, - messageSource.getMessage("server.projects.reference-file.not-found", new Object[] {}, locale)); - refFiles.add(uiReferenceFile); - } + refFiles.add(new UIReferenceFile(join, join.getObject().getFileSize())); } return refFiles; } diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index 702d6722eb4..fc4aeb1cade 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2329,10 +2329,8 @@ ReferenceFile.removeTooltip=Remove reference file from project ReferenceFile.confirmText=Remove reference file {0} from project {1}? ReferenceFile.clickorDrag=Click or drag file to this area to upload ReferenceFile.singleOrMultiple=Supports single or multiple reference file uploads. -ReferenceFile.uploadFileError=Upload failed for reference file {0}. {1} server.projects.reference-file.delete-success=Reference file {0} has been removed from project {1} server.projects.reference-file.delete-error=There was an error removing reference file {0} from project {1} server.projects.reference-file.invalid-content=IRIDA currently supports only DNA reference files. The file you uploaded contained non A, C, G, T, or N bases. Please check that your file does not contain IUPAC ambiguous bases and that your file is a DNA-formatted reference file. server.projects.reference-file.unknown-error=Your reference file failed to upload for an unknown reason. Please try again, or contact an administrator. -server.projects.reference-file.not-found=Error reading file \ No newline at end of file diff --git a/src/main/webapp/resources/js/apis/projects/reference-files.js b/src/main/webapp/resources/js/apis/projects/reference-files.js index 7f0662b94c6..09c4f7f01e1 100644 --- a/src/main/webapp/resources/js/apis/projects/reference-files.js +++ b/src/main/webapp/resources/js/apis/projects/reference-files.js @@ -41,23 +41,3 @@ export async function removeProjectReferenceFile(projectId, fileId) { throw new Error(error.response.data.error); }); } - -/** - * Remove the reference file from the project id provided - * @param {projectId} project for which to remove reference file from - * @param {formData} List of files to upload - * @param config Configuration options for upload - * @return {Promise<*>} `data` contains the OK response; `error` contains error information if an error occurred. - */ -export async function uploadProjectReferenceFiles(projectId, formData, config) { - return await axios - .post( - setBaseUrl(`ajax/reference-files/project/${projectId}`), - formData, - config - ) - .then(({ data }) => data) - .catch((error) => { - throw new Error(error.response.data.error); - }); -} diff --git a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx index c4e298f95de..452698b4dff 100644 --- a/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx +++ b/src/main/webapp/resources/js/pages/projects/reference-files/ReferenceFiles.jsx @@ -1,13 +1,12 @@ import React from "react"; -import { notification, Progress, Space, Table, Typography } from "antd"; +import { notification, Space, Table, Typography } from "antd"; import { InfoAlert } from "../../../components/alerts"; +import { setBaseUrl } from "../../../utilities/url-utilities"; import { downloadProjectReferenceFile, getProjectReferenceFiles, removeProjectReferenceFile, - uploadProjectReferenceFiles, } from "../../../apis/projects/reference-files"; -import { getProjectInfo } from "../../../apis/projects/projects"; import { formatInternationalizedDateTime } from "../../../utilities/date-utilities"; import { ContentLoading } from "../../../components/loader"; @@ -17,6 +16,8 @@ import { } from "../../../components/Buttons"; import { DragUpload } from "../../../components/files/DragUpload"; +import { getProjectInfo } from "../../../apis/projects/projects"; + const { Title } = Typography; /** @@ -29,8 +30,6 @@ export function ReferenceFiles() { const [projectInfo, setProjectInfo] = React.useState(null); const [loading, setLoading] = React.useState(true); - const [progress, setProgress] = React.useState(0); - const pathRegx = new RegExp(/projects\/(\d+)/); const projectId = window.location.pathname.match(pathRegx)[1]; @@ -135,52 +134,27 @@ export function ReferenceFiles() { }); } - // Custom request for drag upload to display progress of uploads - const uploadFiles = async (options) => { - const { onSuccess, onError, file, onProgress } = options; - - const formData = new FormData(); - const config = { - headers: { "content-type": "multipart/form-data" }, - onUploadProgress: (event) => { - const percent = Math.floor((event.loaded / event.total) * 100); - setProgress(percent); - if (percent === 100) { - setTimeout(() => setProgress(0), 1000); - } - onProgress({ percent: (event.loaded / event.total) * 100 }); - }, - }; - formData.append("file", file); - - uploadProjectReferenceFiles(projectId, formData, config) - .then(() => { - onSuccess("Ok"); + // Options for the Ant Design upload component + const referenceFileUploadOptions = { + multiple: true, + accept: ".fasta", + showUploadList: false, + action: setBaseUrl(`ajax/reference-files/project/${projectId}`), + onChange(info) { + const { status } = info.file; + if (status === "done") { notification.success({ message: `${i18n( "ReferenceFile.uploadFileSuccess", - file.name, + info.file.name, projectInfo.projectName )}`, }); updateReferenceFileTable(); - }) - .catch((error) => { - onError("Error"); - notification.error({ - message: i18n( - "ReferenceFile.uploadFileError", - file.name, - error.message - ), - }); - }); - }; - - // Options for the Ant Design upload component - const referenceFileUploadOptions = { - multiple: true, - accept: ".fasta", + } else if (status === "error") { + notification.error({ message: info.file.response.error }); + } + }, }; return ( @@ -190,8 +164,6 @@ export function ReferenceFiles() { {projectInfo && projectInfo.canManage ? ( } - customRequest={uploadFiles} uploadText={i18n("ReferenceFile.clickorDrag")} uploadHint={uploadHintMessage[referenceFileUploadOptions.multiple]} /> diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileAjaxControllerTest.java index b3cf0d73565..123843a9348 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/files/ReferenceFileAjaxControllerTest.java @@ -67,7 +67,7 @@ public void testDeleteReferenceFile() { @Test public void testGetReferenceFiles() { - ResponseEntity> response = controller.getReferenceFilesForProject(PROJECT_ID, Locale.ENGLISH); + ResponseEntity> response = controller.getReferenceFilesForProject(PROJECT_ID); assertEquals("Receive an 200 OK response", response.getStatusCode(), HttpStatus.OK); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectReferenceFileServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectReferenceFileServiceTest.java index f58fa216508..56711a3d3c8 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectReferenceFileServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectReferenceFileServiceTest.java @@ -126,7 +126,7 @@ public void testGetReferenceFileForProject() { when(projectService.read(project.getId())).thenReturn(project); when(referenceFileService.read(file.getId())).thenReturn(file); - uiProjectReferenceFileService.getReferenceFilesForProject(project.getId(), Locale.ENGLISH); + uiProjectReferenceFileService.getReferenceFilesForProject(project.getId()); verify(projectService, times(1)).read(project.getId()); verify(referenceFileService, times(1)).getReferenceFilesForProject(project); From 812a1b86c4c0abf1994668d24b8080b497cf3d34 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 26 Nov 2020 15:28:29 -0600 Subject: [PATCH 164/655] Removed unused path variables --- .../irida/ria/web/files/SequenceFileController.java | 2 -- .../bioinformatics/irida/ria/web/samples/SamplesController.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java index 6cfcfedc106..289fbfc54ca 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.ria.web.files; import java.io.IOException; -import java.nio.file.Path; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -85,7 +84,6 @@ public void downloadSequenceFile(@PathVariable Long sequencingObjectId, @PathVar HttpServletResponse response) throws IOException { SequencingObject sequencingObject = sequencingObjectService.read(sequencingObjectId); SequenceFile sequenceFile = sequencingObject.getFileWithId(sequenceFileId); - Path path = sequenceFile.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + sequenceFile.getLabel() + "\""); sequenceFile.getFileInputStream().transferTo(response.getOutputStream()); response.flushBuffer(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 915dec0e296..2c66b21005a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.ria.web.samples; import java.io.IOException; -import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -353,7 +352,6 @@ public void downloadAssembly(@PathVariable Long sampleId, @PathVariable Long ass Sample sample = sampleService.read(sampleId); GenomeAssembly genomeAssembly = genomeAssemblyService.getGenomeAssemblyForSample(sample, assemblyId); - Path path = genomeAssembly.getFile(); response.setHeader("Content-Disposition", "attachment; filename=\"" + genomeAssembly.getLabel() + "\""); genomeAssembly.getFileInputStream().transferTo(response.getOutputStream()); From f77787cf7540a56e7e9bc0d84b99db6bb371b144 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 26 Nov 2020 16:09:33 -0600 Subject: [PATCH 165/655] Updated getFileInputStream methods to not read into memory but rather stream. Updated usage of getfileiputstream to be used in try-with-resources block so that the underlying inputstream is closed --- .../IridaFileStorageAwsUtilityImpl.java | 10 ++++------ .../IridaFileStorageAzureUtilityImpl.java | 17 +++++------------ .../filesystem/IridaFileStorageUtility.java | 5 ++++- .../irida/ria/utilities/FileUtilities.java | 16 ++++++++++++---- .../web/analysis/AnalysisAjaxController.java | 9 ++++++--- .../ria/web/files/SequenceFileController.java | 9 ++++++++- .../ria/web/samples/SamplesController.java | 8 +++++++- .../services/UIProjectReferenceFileService.java | 9 ++++++++- 8 files changed, 54 insertions(+), 29 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index ddac504e03e..c513cbb78a8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -191,18 +191,16 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { - byte[] bytes = new byte[0]; - try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { - bytes = s3Object.getObjectContent() - .readAllBytes(); + try { + S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + return s3Object.getObjectContent(); } catch (AmazonServiceException e) { logger.error("Couldn't read file from s3 bucket [" + e + "]"); throw new StorageException("Unable to locate file in s3 bucket", e); } catch (Exception e) { logger.error(e.getMessage()); + throw new StorageException("Unable to read file inputstream from s3 bucket", e); } - - return new ByteArrayInputStream(bytes); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index e9adf859297..3370125367a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.ByteArrayInputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -189,20 +188,14 @@ public boolean fileExists(Path file) { */ @Override public InputStream getFileInputStream(Path file) { - byte[] bytes = new byte[0]; + logger.trace("Opening input stream to file on azure [" + file.toString() + "]"); + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); try { - logger.trace("Opening input stream to file on azure [" + file.toString() + "]"); - BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - try (BlobInputStream blobInputStream = blobClient.openInputStream()) { - bytes = blobInputStream.readAllBytes(); - } catch (IOException e) { - logger.error("Couldn't get bytes from file [" + e + "]"); - } + return blobClient.openInputStream(); } catch (BlobStorageException e) { - logger.error("Couldn't read file from azure [" + e + "]"); - throw new StorageException("Unable to locate file on azure", e); + logger.error("Couldn't get file input stream from azure [" + e + "]"); + throw new StorageException("Couldn't get file input stream from azure", e); } - return new ByteArrayInputStream(bytes); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 9a28796456f..0e74f56a777 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -77,7 +77,10 @@ public interface IridaFileStorageUtility { public boolean fileExists(Path file); /** - * Gets the file inputstream + * Gets the file inputstream. + * ### Note: This method must be called in a + * ### try-with-resources block so that the + * ### underlying inputstream is closed. * * @param file The path to the file * @return file inputstream diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java index 0d119f72ffd..d3010a9e663 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java @@ -83,7 +83,11 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re outputStream.putNextEntry(new ZipEntry(zipEntryName.toString())); // 3) COPY all of thy bytes from the file to the output stream. - IOUtils.copy(file.getFileInputStream(),outputStream); + try(InputStream inputStream = file.getFileInputStream()) { + IOUtils.copy(inputStream, outputStream); + } catch (IOException e) { + logger.error("Unable to read input stream from file", e); + } // 4) Close the current entry in the archive in preparation for // the next entry. outputStream.closeEntry(); @@ -151,7 +155,11 @@ public static void createBatchAnalysisOutputFileZippedResponse(HttpServletRespon outputStream.putNextEntry(new ZipEntry(fileName + "/" + outputFilename)); // 3) COPY all of thy bytes from the file to the output stream. - IOUtils.copy(file.getFileInputStream(),outputStream); + try(InputStream inputStream = file.getFileInputStream()) { + IOUtils.copy(inputStream, outputStream); + } catch (IOException e) { + logger.error("Unable to read input stream from file", e); + } // 4) Close the current entry in the archive in preparation for // the next entry. @@ -186,8 +194,8 @@ public static void createSingleFileResponse(HttpServletResponse response, Analys response.setHeader(CONTENT_DISPOSITION, ATTACHMENT_FILENAME + fileName); response.setContentType(CONTENT_TYPE_TEXT); - try (ServletOutputStream outputStream = response.getOutputStream()) { - IOUtils.copy(file.getFileInputStream(), response.getOutputStream()); + try (InputStream inputStream = file.getFileInputStream()) { + IOUtils.copy(inputStream, response.getOutputStream()); } catch (IOException e) { // this generally means that the user has cancelled the download // from their web browser; we can safely ignore this diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 243ee5b9093..432a40d136e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -1073,8 +1073,8 @@ public AnalysisTreeResponse getNewickTree(@PathVariable Long submissionId, Local } else { AnalysisOutputFile file = treeOptional.get(); - try { - List lines = IOUtils.readLines(file.getFileInputStream()); + try(InputStream inputStream = file.getFileInputStream()) { + List lines = IOUtils.readLines(inputStream); if (lines.size() > 0) { tree = lines.get(0); @@ -1094,8 +1094,11 @@ public AnalysisTreeResponse getNewickTree(@PathVariable Long submissionId, Local } } } catch (NoSuchFileException e) { - logger.debug("File was not found: " + e.toString()); + logger.error("File was not found: " + e.toString()); + } catch (IOException e) { + logger.error("Unable to read input stream for file"); } + } return new AnalysisTreeResponse(tree, message); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java index 289fbfc54ca..795c1cd590a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/files/SequenceFileController.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.web.files; import java.io.IOException; +import java.io.InputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -85,7 +86,13 @@ public void downloadSequenceFile(@PathVariable Long sequencingObjectId, @PathVar SequencingObject sequencingObject = sequencingObjectService.read(sequencingObjectId); SequenceFile sequenceFile = sequencingObject.getFileWithId(sequenceFileId); response.setHeader("Content-Disposition", "attachment; filename=\"" + sequenceFile.getLabel() + "\""); - sequenceFile.getFileInputStream().transferTo(response.getOutputStream()); + + try(InputStream inputStream = sequenceFile.getFileInputStream()) { + inputStream.transferTo(response.getOutputStream()); + } catch (IOException e) { + logger.error("Unable to read input stream from file", e); + } + response.flushBuffer(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index 2c66b21005a..81b186b09d1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.web.samples; import java.io.IOException; +import java.io.InputStream; import java.util.*; import java.util.stream.Collectors; @@ -354,7 +355,12 @@ public void downloadAssembly(@PathVariable Long sampleId, @PathVariable Long ass response.setHeader("Content-Disposition", "attachment; filename=\"" + genomeAssembly.getLabel() + "\""); - genomeAssembly.getFileInputStream().transferTo(response.getOutputStream()); + try(InputStream inputStream = genomeAssembly.getFileInputStream()) { + inputStream.transferTo(response.getOutputStream()); + } catch (IOException e) { + logger.error("Unable to read input stream from file", e); + } + response.flushBuffer(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java index 158591f96a6..132316fea7f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.ria.web.services; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -120,7 +121,13 @@ public String deleteReferenceFile(Long fileId, Long projectId, Locale locale) th public void downloadReferenceFile(Long fileId, HttpServletResponse response) throws IOException { ReferenceFile file = referenceFileService.read(fileId); response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getLabel() + "\""); - file.getFileInputStream().transferTo(response.getOutputStream()); + + try(InputStream inputStream = file.getFileInputStream()) { + inputStream.transferTo(response.getOutputStream()); + } catch (IOException e) { + logger.error("Unable to read input stream from file", e); + } + response.flushBuffer(); } From dca4590ac2b04621e4f52f20200fe2bc9c40266a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 30 Nov 2020 14:47:04 -0600 Subject: [PATCH 166/655] Added throwable cause to error message --- .../irida/ria/web/analysis/AnalysisAjaxController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 432a40d136e..95314a0e773 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -1096,7 +1096,7 @@ public AnalysisTreeResponse getNewickTree(@PathVariable Long submissionId, Local } catch (NoSuchFileException e) { logger.error("File was not found: " + e.toString()); } catch (IOException e) { - logger.error("Unable to read input stream for file"); + logger.error("Unable to read input stream for file", e); } } From 84c6da44de59bb99abc66d96d8d8915d0632db45 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 3 Dec 2020 15:44:25 -0600 Subject: [PATCH 167/655] Updated getOutputFile to use the iridafilestorageutility to retrieve requsted chunk of text from output file. Also, updated this method to use the inputstream of a file to read lines --- .../IridaFileStorageAwsUtilityImpl.java | 9 +-- .../IridaFileStorageAzureUtilityImpl.java | 12 ++-- .../IridaFileStorageLocalUtilityImpl.java | 11 ++- .../filesystem/IridaFileStorageUtility.java | 6 +- .../ajax/dto/analysis/FileChunkResponse.java | 31 ++++++++ .../web/analysis/AnalysisAjaxController.java | 70 +++++-------------- .../bioinformatics/irida/util/IridaFiles.java | 1 + .../AnalysesTable/AnalysisDownloadButton.jsx | 19 ++--- .../components/AnalysisTextPreview.jsx | 25 ++++--- 9 files changed, 98 insertions(+), 86 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/analysis/FileChunkResponse.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 21f0ef1c9d9..642b0ff2e8b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -20,6 +20,8 @@ import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; +import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.amazonaws.AmazonServiceException; @@ -338,17 +340,16 @@ public Long getFileSizeBytes(Path file) { * {@inheritDoc} */ @Override - public String readChunk(Path file, Long seek, Long chunk) { - byte[] bytes = new byte[seek.intValue() + chunk.intValue()]; + public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { // The range of bytes to read. Start at seek and get `chunk` amount of bytes from seek point GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, getAwsFileAbsolutePath(file)).withRange( seek, chunk); try (S3Object s3Object = s3.getObject(rangeObjectRequest); S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { - bytes = s3ObjectInputStream.readAllBytes(); + return new FileChunkResponse(new String(s3ObjectInputStream.readAllBytes()), seek+chunk); } catch (IOException e) { logger.error("Couldn't get chunk from s3 bucket", e); } - return new String(bytes); + return null; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 7bbbb4e63a9..d84a0503788 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -25,6 +25,9 @@ import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; +import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.azure.storage.blob.BlobClient; @@ -339,18 +342,17 @@ public Long getFileSizeBytes(Path file) { * {@inheritDoc} */ @Override - public String readChunk(Path file, Long seek, Long chunk) { - byte[] bytes = new byte[(seek.intValue() + chunk.intValue())]; + public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); // The range of bytes to read. Start at seek and get `chunk` amount of bytes from seek point BlobRange blobRange = new BlobRange(seek, chunk); try(BlobInputStream blobInputStream = blobClient.openInputStream(blobRange, null)){ - bytes = blobInputStream.readAllBytes(); + return new FileChunkResponse(new String(blobInputStream.readAllBytes()), seek+chunk); } catch (BlobStorageException e) { logger.error("Couldn't find file on azure", e); } catch (IOException e) { - logger.error("Couldn't read file from azure", e); + logger.error("Unable to read chunk from azure", e); } - return new String(bytes); + return null; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 1f4be6407f5..eb3d04de747 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -23,6 +23,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; +import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.util.FileUtils; /** @@ -245,17 +247,14 @@ public Long getFileSizeBytes(Path file) { } @Override - public String readChunk(Path file, Long seek, Long chunk) { - String text = ""; + public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { try { final RandomAccessFile randomAccessFile = new RandomAccessFile(file.toFile(), "r"); randomAccessFile.seek(seek); - // add aws code here for reading chunk - text = FileUtilities.readChunk(randomAccessFile, seek, chunk); - + return new FileChunkResponse(FileUtilities.readChunk(randomAccessFile, seek, chunk), randomAccessFile.getFilePointer()); } catch (IOException e ) { logger.error("Could not read output file ", e); } - return text; + return null; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 3a92635f4c0..c434883f298 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -7,6 +7,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; +import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import com.google.common.collect.Lists; @@ -138,8 +140,8 @@ public interface IridaFileStorageUtility { * @param file The {@link Path} to the file * @param seek File pointer to where to start reading from * @param chunk Size in bytes to read from seek point - * @return {@link String} The text from the file in chunks + * @return {@link FileChunkResponse} Response dto containing the text and file pointer */ - public String readChunk(Path file, Long seek, Long chunk); + public FileChunkResponse readChunk(Path file, Long seek, Long chunk); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/analysis/FileChunkResponse.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/analysis/FileChunkResponse.java new file mode 100644 index 00000000000..dc323cafad9 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/analysis/FileChunkResponse.java @@ -0,0 +1,31 @@ +package ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis; + +/** + * Used to send the text from the chunk requested to the UI + */ + +public class FileChunkResponse { + private String text; + private Long filePointer; + + public FileChunkResponse(String text, Long filePointer) { + this.text = text; + this.filePointer = filePointer; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public Long getFilePointer() { + return filePointer; + } + + public void setFilePointer(Long filePointer) { + this.filePointer = filePointer; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index dbd4fb82d42..a30c3b25221 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -47,6 +47,7 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.UpdatedAnalysisProgress; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; import ca.corefacility.bioinformatics.irida.ria.web.analysis.auditing.AnalysisAudit; import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.*; import ca.corefacility.bioinformatics.irida.ria.web.components.AnalysisOutputFileDownloadManager; @@ -510,7 +511,7 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable final AnalysisOutputFile aof = analysisOutputFile.get(); final Path aofFile = aof.getFile(); final ToolExecution tool = aof.getCreatedByTool(); - final AnalysisOutputFileInfo contents = new AnalysisOutputFileInfo(); + AnalysisOutputFileInfo contents = new AnalysisOutputFileInfo(); contents.setId(aof.getId()); contents.setAnalysisSubmissionId(submission.getId()); contents.setAnalysisId(analysis.getId()); @@ -522,58 +523,25 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable contents.setToolName(tool.getToolName()); contents.setToolVersion(tool.getToolVersion()); - IridaTemporaryFile iridaTemporaryFile = null; - RandomAccessFile randomAccessFile; - /* - * Cloud storage doesn't have ability to read files by lines so we need to download the file and access it. - * If reading chunks of a file then we don't download the file but rather just get the bytes from the inputstream - * as a string. - */ - try { - if (seek == 0) { - if (chunk != null && chunk > 0) { - contents.setText(iridaFileStorageUtility.readChunk(aof.getFile(), seek, chunk)); - contents.setChunk(chunk); - contents.setStartSeek(seek); - contents.setFilePointer(seek+chunk); - } else { - iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(aofFile); - randomAccessFile = new RandomAccessFile(iridaTemporaryFile.getFile().toFile(), "r"); - randomAccessFile.seek(seek); - final BufferedReader reader = new BufferedReader(new FileReader(randomAccessFile.getFD())); - final List lines = FileUtilities.readLinesLimit(reader, limit, start, end); - contents.setLines(lines); - contents.setLimit((long) lines.size()); - contents.setStart(start); - contents.setEnd(start + lines.size()); - } - } else { - if (chunk != null && chunk > 0) { - contents.setText(iridaFileStorageUtility.readChunk(aof.getFile(), seek, chunk)); - contents.setChunk(chunk); - contents.setStartSeek(seek); - contents.setFilePointer(seek+chunk); - } else { - iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(aofFile); - randomAccessFile = new RandomAccessFile(iridaTemporaryFile.getFile().toFile(), "r"); - randomAccessFile.seek(seek); - final List lines = FileUtilities.readLinesFromFilePointer(randomAccessFile, limit); - contents.setLines(lines); - contents.setStartSeek(seek); - contents.setStart(start); - contents.setLimit((long) lines.size()); - } - } - - } catch (IOException e) { - logger.error("Could not read output file '" + aof.getId() + "' " + e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - contents.setError("Could not read output file"); - } finally { - if(iridaTemporaryFile != null ) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + if (chunk != null && chunk > 0) { + FileChunkResponse fileChunkResponse = iridaFileStorageUtility.readChunk(aof.getFile(), seek, chunk); + contents.setText(fileChunkResponse.getText()); + contents.setChunk(chunk); + contents.setStartSeek(seek); + contents.setFilePointer(fileChunkResponse.getFilePointer()); + } else { + try(BufferedReader reader = new BufferedReader(new InputStreamReader(aof.getFileInputStream(), "UTF-8"))) { + List lines = FileUtilities.readLinesLimit(reader, limit, start, end); + contents.setLines(lines); + contents.setLimit((long) lines.size()); + contents.setStart(start); + contents.setEnd(start + lines.size()); + contents.setFilePointer(start + lines.size()); + } catch (IOException e) { + logger.error("Could not read output file stream'" + aof.getId() + "' " + e); } } + return contents; } else { response.setStatus(HttpServletResponse.SC_NOT_FOUND); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 0ef2db99924..17d6843d44e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -95,4 +95,5 @@ public static Long getFileSizeBytes(Path file) { * @return if file exists or not */ public static boolean fileExists(Path file) { return iridaFileStorageUtility.fileExists(file);} + } diff --git a/src/main/webapp/resources/js/components/AnalysesTable/AnalysisDownloadButton.jsx b/src/main/webapp/resources/js/components/AnalysesTable/AnalysisDownloadButton.jsx index 9138ef20632..339ee252fcb 100644 --- a/src/main/webapp/resources/js/components/AnalysesTable/AnalysisDownloadButton.jsx +++ b/src/main/webapp/resources/js/components/AnalysesTable/AnalysisDownloadButton.jsx @@ -15,20 +15,24 @@ import { AnalysesTableContext } from "../../contexts/AnalysesTableContext"; * @constructor */ export function AnalysisDownloadButton({ state, analysisId, updateDelay }) { - const [isAnalysisCompleted, setIsAnalysisCompleted] = useState(state.value !== "COMPLETED"); + const [isAnalysisCompleted, setIsAnalysisCompleted] = useState( + state.value !== "COMPLETED" + ); const { analysesTableContext } = useContext(AnalysesTableContext); // Update if an analysis is completed (to enable or disable download results button) const intervalId = useInterval(() => { - if(state.value !== "COMPLETED" && state.value !== "ERROR") { - let rowData = analysesTableContext.rows.filter(row => row.identifier === analysisId); + if (state.value !== "COMPLETED" && state.value !== "ERROR") { + let rowData = analysesTableContext.rows.filter( + (row) => row.identifier === analysisId + ); let currRowData = rowData[rowData.length - 1]; - if(typeof currRowData !== "undefined") { - if(isAnalysisCompleted !== currRowData.isCompleted) { + if (typeof currRowData !== "undefined") { + if (isAnalysisCompleted !== currRowData.isCompleted) { setIsAnalysisCompleted(currRowData.isCompleted); } - if(currRowData.isCompleted || currRowData.isError) { + if (currRowData.isCompleted || currRowData.isError) { clearInterval(intervalId); } } @@ -39,7 +43,7 @@ export function AnalysisDownloadButton({ state, analysisId, updateDelay }) { return ( ); - } diff --git a/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx b/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx index 6f79281647a..95f51eaa85e 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx @@ -3,7 +3,7 @@ */ import React, { useEffect, useState } from "react"; -import { Divider, Row, Typography } from "antd"; +import { Divider, Space, Typography } from "antd"; import { getDataViaChunks } from "../../../apis/analysis/analysis"; import { ContentLoading } from "../../../components/loader/ContentLoading"; import { @@ -66,8 +66,9 @@ export default function AnalysisTextPreview({ output }) { if ( scollElement.scrollTop + scrollableDivHeight >= - scollElement.scrollHeight && - getNewChunkSize(filePointer, output.fileSizeBytes, chunkSize) >= 0 + scollElement.scrollHeight - 1 && + getNewChunkSize(filePointer, output.fileSizeBytes, chunkSize) >= 0 && + filePointer < output.fileSizeBytes - 1 ) { setLoading(true); getDataViaChunks({ @@ -105,13 +106,17 @@ export default function AnalysisTextPreview({ output }) { > {fileRows} -
    -
    - {loading ? ( - - ) : null} +
    + + + + {loading ? ( + + ) : null} + +
    From 2877746336e884226e0d11e4e41c6eb88596f042 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 3 Dec 2020 18:52:06 -0600 Subject: [PATCH 168/655] Updated addFirstLine method to use inputstreamreader rather than using randomaccessfile as we use the inputstreamreader with the bufferedreader to read the file inputstream. Updated filepointers when getting chunks through the irida storage utility classes for object storage --- .../IridaFileStorageAwsUtilityImpl.java | 9 ++++-- .../IridaFileStorageAzureUtilityImpl.java | 11 +++++-- .../web/analysis/AnalysisAjaxController.java | 32 +++++++------------ 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 642b0ff2e8b..66f004cb4e1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -341,12 +341,17 @@ public Long getFileSizeBytes(Path file) { */ @Override public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { - // The range of bytes to read. Start at seek and get `chunk` amount of bytes from seek point + /* + The range of bytes to read. Start at seek and get `chunk` amount of bytes from seek point. + However a smaller amount of bytes may be read, so we set the file pointer accordingly + */ GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, getAwsFileAbsolutePath(file)).withRange( seek, chunk); try (S3Object s3Object = s3.getObject(rangeObjectRequest); S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { - return new FileChunkResponse(new String(s3ObjectInputStream.readAllBytes()), seek+chunk); + // Read the bytes of the retrieved s3ObjectInputStream chunk + byte[] bytes = s3ObjectInputStream.readAllBytes(); + return new FileChunkResponse(new String(bytes), seek + (bytes.length - 1)); } catch (IOException e) { logger.error("Couldn't get chunk from s3 bucket", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index d84a0503788..729db47fbe2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -344,10 +344,15 @@ public Long getFileSizeBytes(Path file) { @Override public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - // The range of bytes to read. Start at seek and get `chunk` amount of bytes from seek point + /* + The range of bytes to read. Start at seek and get `chunk` amount of bytes from seek point. + However a smaller amount of bytes may be read, so we set the file pointer accordingly + */ BlobRange blobRange = new BlobRange(seek, chunk); - try(BlobInputStream blobInputStream = blobClient.openInputStream(blobRange, null)){ - return new FileChunkResponse(new String(blobInputStream.readAllBytes()), seek+chunk); + try (BlobInputStream blobInputStream = blobClient.openInputStream(blobRange, null)) { + // Read the bytes of the retrieved blobInputStream chunk + byte[] bytes = blobInputStream.readAllBytes(); + return new FileChunkResponse(new String(bytes), seek + (bytes.length - 1)); } catch (BlobStorageException e) { logger.error("Couldn't find file on azure", e); } catch (IOException e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index a30c3b25221..0b30c07afa7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -458,26 +458,15 @@ private AnalysisOutputFileInfo getAnalysisOutputFileInfo(AnalysisSubmission subm * @param aof {@link AnalysisOutputFile} to read from */ private void addFirstLine(AnalysisOutputFileInfo info, AnalysisOutputFile aof) { - RandomAccessFile reader = null; final Path aofFile = aof.getFile(); - try { - IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(aofFile); - reader = new RandomAccessFile(iridaTemporaryFile.getFile().toFile(), "r"); - info.setFirstLine(reader.readLine()); - info.setFilePointer(reader.getFilePointer()); - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); - } catch (FileNotFoundException e) { - logger.error("Could not find file '" + aofFile + "' " + e); + + try(BufferedReader reader = new BufferedReader(new InputStreamReader(aof.getFileInputStream(), "UTF-8"))) { + String firstLineText = reader.readLine(); + info.setFirstLine(firstLineText); + // Set the pointer to the beginning of the next line. + info.setFilePointer(Long.valueOf(firstLineText.getBytes().length) + 1); } catch (IOException e) { - logger.error("Could not read file '" + aofFile + "' " + e); - } finally { - try { - if (reader != null) { - reader.close(); - } - } catch (IOException e) { - logger.error("Could not close file handle for '" + aofFile + "' " + e); - } + logger.error("Could not get file input stream '" + aofFile + "' " + e); } } @@ -524,16 +513,19 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable contents.setToolVersion(tool.getToolVersion()); if (chunk != null && chunk > 0) { + // Read the requested chunk from the iridafilestorageutility and set the required fields of the contents object FileChunkResponse fileChunkResponse = iridaFileStorageUtility.readChunk(aof.getFile(), seek, chunk); contents.setText(fileChunkResponse.getText()); contents.setChunk(chunk); contents.setStartSeek(seek); contents.setFilePointer(fileChunkResponse.getFilePointer()); } else { + // Read the inputstream and get the lines requested of the output file and set the required fields of the contents object try(BufferedReader reader = new BufferedReader(new InputStreamReader(aof.getFileInputStream(), "UTF-8"))) { - List lines = FileUtilities.readLinesLimit(reader, limit, start, end); + List lines = new ArrayList<>(); + lines.addAll(FileUtilities.readLinesLimit(reader, limit, start, end)); contents.setLines(lines); - contents.setLimit((long) lines.size()); + contents.setLimit((long) lines.size() - 1); contents.setStart(start); contents.setEnd(start + lines.size()); contents.setFilePointer(start + lines.size()); From 00ae52fa3cccd0509608f4e8ee70243872192a87 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 3 Dec 2020 19:04:29 -0600 Subject: [PATCH 169/655] Removed unused imports and refactored to use try-with-resources --- .../irida/processing/impl/ChecksumFileProcessor.java | 1 - .../irida/processing/impl/GzipFileProcessor.java | 5 +---- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 1 - .../filesystem/IridaFileStorageAzureUtilityImpl.java | 3 --- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 2 -- 5 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index d1371a853bc..f556133f743 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -59,7 +59,6 @@ public void process(SequencingObject sequencingObject) { logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); fileRepository.saveMetadata(file); - is.close(); } catch (IOException e) { throw new FileProcessorException("could not calculate checksum", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java index c2dfd1dd69a..c99a4dda1b5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/GzipFileProcessor.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.processing.impl; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -120,8 +119,7 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc file = addExtensionToFilename(file, GZIP_EXTENSION); sequenceFile.setFile(file); - InputStream is = sequenceFile.getFileInputStream(); - try (GZIPInputStream zippedInputStream = new GZIPInputStream(is)) { + try (GZIPInputStream zippedInputStream = new GZIPInputStream(sequenceFile.getFileInputStream())) { logger.trace("Handling gzip compressed file."); Path targetDirectory = Files.createTempDirectory(null); @@ -130,7 +128,6 @@ public void processSingleFile(SequenceFile sequenceFile) throws FileProcessorExc logger.debug("Writing uncompressed file to [" + target + "]"); Files.copy(zippedInputStream, target); - is.close(); sequenceFile.setFile(target); sequenceFile = sequenceFileRepository.save(sequenceFile); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 66f004cb4e1..30785be5e98 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -21,7 +21,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.amazonaws.AmazonServiceException; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 729db47fbe2..388a5e01c57 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -5,7 +5,6 @@ import java.io.InputStream; import java.nio.channels.FileChannel; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -25,9 +24,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.azure.storage.blob.BlobClient; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index eb3d04de747..4ba135e16d5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.repositories.filesystem; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; @@ -24,7 +23,6 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.util.FileUtils; /** From ae30b2611a972dd1fc608e18e4191826ae08a151 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 3 Dec 2020 19:14:49 -0600 Subject: [PATCH 170/655] Removed extra lines added, general refactoring --- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 4 +--- .../repositories/filesystem/IridaFileStorageUtility.java | 1 - .../bioinformatics/irida/ria/utilities/FileUtilities.java | 1 - .../irida/ria/web/analysis/AnalysisAjaxController.java | 4 ++-- .../irida/ria/web/samples/SamplesController.java | 2 -- 5 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 4ba135e16d5..8a65976a1b9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -149,10 +149,8 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { byte[] bytes = new byte[2]; is.read(bytes); - boolean gzipped = ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); - is.close(); - return gzipped; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index c434883f298..c2274036337 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -8,7 +8,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import com.google.common.collect.Lists; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java index b8645ba96fe..d3010a9e663 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java @@ -88,7 +88,6 @@ public static void createAnalysisOutputFileZippedResponse(HttpServletResponse re } catch (IOException e) { logger.error("Unable to read input stream from file", e); } - // 4) Close the current entry in the archive in preparation for // the next entry. outputStream.closeEntry(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 0b30c07afa7..004c089da27 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -500,7 +500,7 @@ public AnalysisOutputFileInfo getOutputFile(@PathVariable Long id, @PathVariable final AnalysisOutputFile aof = analysisOutputFile.get(); final Path aofFile = aof.getFile(); final ToolExecution tool = aof.getCreatedByTool(); - AnalysisOutputFileInfo contents = new AnalysisOutputFileInfo(); + final AnalysisOutputFileInfo contents = new AnalysisOutputFileInfo(); contents.setId(aof.getId()); contents.setAnalysisSubmissionId(submission.getId()); contents.setAnalysisId(analysis.getId()); @@ -888,7 +888,7 @@ public Map getNewickForAnalysis(@PathVariable Long submissionId) if (treeFileForSubmission.isPresent()) { AnalysisOutputFile file = treeFileForSubmission.get(); - List lines = IOUtils.readLines(IridaFiles.getFileInputStream(file.getFile())); + List lines = IOUtils.readLines(file.getFileInputStream()); return ImmutableMap.of("newick", lines.get(0)); } else { throw new IOException("Newick file could not be found for this submission"); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java index bc3b7103128..81b186b09d1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesController.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.InputStream; - import java.util.*; import java.util.stream.Collectors; @@ -356,7 +355,6 @@ public void downloadAssembly(@PathVariable Long sampleId, @PathVariable Long ass response.setHeader("Content-Disposition", "attachment; filename=\"" + genomeAssembly.getLabel() + "\""); - try(InputStream inputStream = genomeAssembly.getFileInputStream()) { inputStream.transferTo(response.getOutputStream()); } catch (IOException e) { From 199c63efc9c1138264e8b5cb202ae69f7a54d73c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 4 Dec 2020 09:10:58 -0600 Subject: [PATCH 171/655] Updated getting sistr results from output file to use inputstream rather than a filereader: --- .../web/analysis/AnalysisAjaxController.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 004c089da27..6715410cd68 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -44,7 +44,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.ProjectAnalysisSubmissionJoin; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.UpdatedAnalysisProgress; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -61,8 +60,6 @@ import ca.corefacility.bioinformatics.irida.service.user.UserService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; - import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonMappingException; @@ -733,14 +730,13 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { if (analysisTypesService.getViewerForAnalysisType(analysisType).get().equals("sistr")) { Analysis analysis = submission.getAnalysis(); - Path path = null; + Path path; if(analysis.getAnalysisOutputFile(sistrFileKey) != null) { path = analysis.getAnalysisOutputFile(sistrFileKey).getFile(); - IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(path); - try { - String json = new Scanner(new BufferedReader(new FileReader(iridaTemporaryFile.getFile().toFile()))).useDelimiter("\\Z") - .next(); + try(InputStream inputStream = iridaFileStorageUtility.getFileInputStream(path)) { + String json = new Scanner(inputStream).useDelimiter("\\Z") + .next(); // verify file is proper json file and map to a SistrResult list ObjectMapper mapper = new ObjectMapper(); List sistrResults = mapper.readValue(json, new TypeReference>() { @@ -757,14 +753,11 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { } else { logger.error("SISTR results for file [" + path + "] are not correctly formatted"); } - } catch (FileNotFoundException e) { - logger.error("File [" + path + "] not found", e); } catch (JsonParseException | JsonMappingException e) { logger.error("Error attempting to parse file [" + path + "] as JSON", e); } catch (IOException e) { - logger.error("Error reading file [" + path + "]", e); + logger.error("Error reading file input stream [" + path + "]", e); } - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); } else { logger.error("Null response from analysis.getAnalysisOutputFile(sistrFileKey). " + "No output file was found for the default sistrFileKey \""+sistrFileKey + "\". "+ From 5cfee4fb80cb9f2b9179d752ba820f4a8fc08450 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 4 Dec 2020 15:28:12 -0600 Subject: [PATCH 172/655] Removed duplicate setting of setLoading to false --- .../js/pages/analysis/components/AnalysisTextPreview.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx b/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx index 538611f3121..95f51eaa85e 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/AnalysisTextPreview.jsx @@ -84,7 +84,6 @@ export default function AnalysisTextPreview({ output }) { document.getElementById( `${output.filename}-preview-status` ).innerText = fileSizeLoaded(data.filePointer, output.fileSizeBytes); - setLoading(false); } setLoading(false); }); From a4df754dfcaeb5ffdb31b0854a76c2b7ff2c9c31 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 7 Dec 2020 09:30:00 -0600 Subject: [PATCH 173/655] Removed unused method and imports --- .../bioinformatics/irida/util/FileUtils.java | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java index fa9d8a403d9..e647eb70ac9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/FileUtils.java @@ -4,18 +4,14 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.zip.GZIPInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A class containing a number of utilities for dealing with files. */ public class FileUtils { - private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); /** * Determines if a file is compressed. Adapted from stackoverflow answer: @@ -52,44 +48,4 @@ public static String humanReadableByteCount(long bytes, boolean si) { String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); } - - /** - * Removes temporarily downloaded files from an object store off the local filesystem - * and the parent directory if it is empty - * - * @param fileToRemove The file to remove - */ - public static void removeTemporaryFile(Path fileToRemove){ - - String parentDirectoryPath = fileToRemove.getParent().toString(); - Path dirToRemovePath = Paths.get(parentDirectoryPath); - - if(Files.isRegularFile(fileToRemove)) { - try { - Files.delete(fileToRemove); - } catch (IOException e) { - logger.debug("Unable to find file to remove " + e); - } - } - - if(Files.isDirectory(dirToRemovePath)) { - boolean directoryIsNumeric = dirToRemovePath.toString() - .substring(dirToRemovePath.toString() - .length() - 1) - .matches("\\d+"); - // Directory is empty - if (directoryIsNumeric && dirToRemovePath.toFile() - .listFiles().length == 0) { - try { - org.apache.commons.io.FileUtils.deleteDirectory(dirToRemovePath.toFile()); - // Recusrsively call remoteTemporaryFile to remove all - // other empty numeric named parent directories - removeTemporaryFile(dirToRemovePath); - } catch (IOException e) { - logger.debug("Unable to remove directory " + e); - } - } - } - } - } From 7f222c3a7b229f4887431db73e5b7d79e8a22380 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 7 Dec 2020 09:34:16 -0600 Subject: [PATCH 174/655] Moved code block into a try with resources block so that the underlying inputstream is closed --- .../irida/ria/web/analysis/AnalysisAjaxController.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 6715410cd68..f5defdabfcc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -881,8 +881,12 @@ public Map getNewickForAnalysis(@PathVariable Long submissionId) if (treeFileForSubmission.isPresent()) { AnalysisOutputFile file = treeFileForSubmission.get(); - List lines = IOUtils.readLines(file.getFileInputStream()); - return ImmutableMap.of("newick", lines.get(0)); + try(InputStream inputStream = file.getFileInputStream()) { + List lines = IOUtils.readLines(inputStream); + return ImmutableMap.of("newick", lines.get(0)); + } catch (IOException e) { + throw new IOException("Unable to read file input stream. ", e); + } } else { throw new IOException("Newick file could not be found for this submission"); } From ed6ba575a18a85dc27abce2c4b546dfc91efbc3f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 8 Dec 2020 14:39:12 -0600 Subject: [PATCH 175/655] Added logic to check if a connection to a cloud provider is successful or not and added logic to check if the local filesystem is writeable. If these fail then we exit and log the exception --- .../IridaApiFilesystemRepositoryConfig.java | 35 +++++++++++++++++-- .../services/IridaApiServicesConfig.java | 14 ++++++-- .../irida/model/enums/StorageType.java | 22 ++++++++++++ .../IridaFileStorageAwsUtilityImpl.java | 16 +++++++++ .../IridaFileStorageAzureUtilityImpl.java | 20 +++++++++++ .../IridaFileStorageLocalUtilityImpl.java | 7 ++++ .../filesystem/IridaFileStorageUtility.java | 10 ++++++ 7 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java index af900b28503..456956eb92b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java @@ -1,10 +1,13 @@ package ca.corefacility.bioinformatics.irida.config.repository; import ca.corefacility.bioinformatics.irida.model.assembly.UploadedAssembly; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.FilesystemSupplementedRepositoryImpl.RelativePathTranslatorListener; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -45,6 +48,10 @@ public class IridaApiFilesystemRepositoryConfig { @Autowired private ApplicationContext applicationContext; + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + + @Bean public RelativePathTranslatorListener relativePathTranslatorListener( final @Qualifier("referenceFileBaseDirectory") Path referenceFileBaseDirectory, @@ -91,15 +98,37 @@ public Path assemblyFileBaseDirectory() throws IOException { return getExistingPathOrThrow(assemblyFileBaseDirectory); } - //FIXME Update the code to check if the connection to a cloud provider - // is successful or not as well as check if path is writeable for local file storage private Path getExistingPathOrThrow(String directory) { Path baseDirectory = Paths.get(directory); - if (!Files.exists(baseDirectory) && storageType.equals("local")) { + if (!Files.exists(baseDirectory) && storageType.equalsIgnoreCase(StorageType.LOCAL.toString())) { throw new IllegalStateException( String.format("Cannot continue startup; base directory [%s] does not exist!", baseDirectory.toString())); } else { + if(storageType.equalsIgnoreCase(StorageType.LOCAL.toString())) { + try { + // Check if basedirectory path is writeable by creating a temp file and then removing it + Path tempFile = Files.createTempFile(baseDirectory, "", ""); + // Check if directory is writeable + boolean directoryWriteable = Files.isWritable(tempFile); + + try { + // Cleanup the temp file created in the directory + Files.delete(tempFile); + } catch (IOException e) { + logger.error("An I/O error occurred while attempting to remove temp file ", e); + } + + if(!directoryWriteable) { + // Log the error and exit so startup does not continue + logger.error("Cannot continue startup; base directory " + baseDirectory + " does not have write access! Please check directory permissions."); + System.exit(1); + } + } catch (IOException e) { + logger.error("Unable to create temporary file. Please check directory permissions", e); + System.exit(1); + } + } logger.info(String.format( "Using specified existing directory at [%s]. The directory *will not* be removed at shutdown time.", baseDirectory.toString())); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 8c13d7f66f4..1a1f00dcfd1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -8,6 +8,7 @@ import ca.corefacility.bioinformatics.irida.config.services.conditions.NreplServerSpringCondition; import ca.corefacility.bioinformatics.irida.config.services.scheduled.IridaScheduledTasksConfig; import ca.corefacility.bioinformatics.irida.config.workflow.IridaWorkflowsConfig; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.plugins.IridaPlugin; @@ -321,14 +322,23 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageUtility") public IridaFileStorageUtility iridaFileStorageService() { IridaFileStorageUtility iridaFileStorageUtility; - if (storageType.equalsIgnoreCase("aws")) { + if (storageType.equalsIgnoreCase(StorageType.AWS.toString())) { iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); - } else if(storageType.equalsIgnoreCase("azure")) { + } else if (storageType.equalsIgnoreCase(StorageType.AZURE.toString())) { iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(containerUrl, sasToken, containerName); } else { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); } + + // Check if connection is valid to local file system or cloud provider + try { + iridaFileStorageUtility.checkConnectivity(); + } catch (IllegalStateException e) { + // Log the error and exit so startup does not continue + logger.error("Unable to start up IRIDA! ", e); + System.exit(1); + } IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); return iridaFileStorageUtility; } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java new file mode 100644 index 00000000000..faa9f7e976c --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java @@ -0,0 +1,22 @@ +package ca.corefacility.bioinformatics.irida.model.enums; + +/** + * Defines a set of storage types. + * + */ + +public enum StorageType { + LOCAL("local"), + AWS("aws"), + AZURE("azure"); + + private String storageType; + + private StorageType(String storageType) { + this.storageType = storageType; + } + + public String toString() { + return storageType; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index c513cbb78a8..354a951501a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -308,4 +308,20 @@ public byte[] readAllBytes(Path file) { } return bytes; } + + @Override + public boolean checkConnectivity() throws IllegalStateException { + try { + if (s3.doesBucketExistV2(bucketName)) { + logger.debug("Successfully connected to aws s3 bucket ", bucketName); + return true; + } else { + throw new IllegalStateException( + "Unable to connect to aws s3 bucket. Please check that your credentials are valid and that the bucket " + bucketName + " exists."); + } + } catch (AmazonServiceException e) { + throw new IllegalStateException( + "Unable to connect to aws s3 bucket. Please check that your credentials are valid and that the bucket " + bucketName + " exists."); + } + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 3370125367a..5b664182085 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -38,12 +38,15 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; + private String containerName; + @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl) .sasToken(sasToken) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + this.containerName = containerName; } /** @@ -305,4 +308,21 @@ private String getAzureFileAbsolutePath(Path file) { } return absolutePath; } + + @Override + public boolean checkConnectivity() throws IllegalStateException { + try { + // Make a api request to get container properties + containerClient.getProperties(); + logger.debug("Successfully connected to azure container ", containerName); + return true; + } catch (Exception e) { + /* + Throw an exception which is caught at startup advising the user that the + connection was not successful to the azure container + */ + throw new IllegalStateException( + "Unable to connect to azure container. Please check that your credentials are valid and that the container " + containerName + " exists."); + } + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index ec948d98b42..b35f6b92a43 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -220,4 +220,11 @@ public byte[] readAllBytes(Path file) { } return bytes; } + + @Override + public boolean checkConnectivity() { + // Since it's the local filesystem we just return true + return true; + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 0e74f56a777..7a22dec7ba9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -123,4 +123,14 @@ public interface IridaFileStorageUtility { * @return the bytes for the file */ public byte[] readAllBytes(Path file); + + /** + * Check the connectivity to the file storage + * mechanism. For cloud services we must make an + * api call to check if containers/buckets exist. + * + * @return Connection valid or not + * @throws IllegalStateException if connection failed + */ + public boolean checkConnectivity() throws IllegalStateException; } From 470fda92c3c20dc51cb416978b610f2ad969aebf Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 8 Dec 2020 15:06:56 -0600 Subject: [PATCH 176/655] Added javadoc --- .../bioinformatics/irida/model/enums/StorageType.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java index faa9f7e976c..520ac9598f6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java @@ -16,6 +16,10 @@ private StorageType(String storageType) { this.storageType = storageType; } + /** + * {@inheritDoc} + */ + @Override public String toString() { return storageType; } From d18eeb0bc2be7ea385f8327358b793ce24e506da Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 9 Dec 2020 13:19:51 -0600 Subject: [PATCH 177/655] Updated views to use a try with resources when getting an inputstream --- .../api/RESTAnalysisSubmissionController.java | 17 +++++++++++------ .../irida/web/spring/view/CSVView.java | 16 ++++++++++------ .../irida/web/spring/view/FastaView.java | 16 ++++++++++------ .../irida/web/spring/view/FastqView.java | 16 ++++++++++------ .../irida/web/spring/view/GenbankView.java | 16 ++++++++++------ .../irida/web/spring/view/NewickFileView.java | 16 ++++++++++------ 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index cf40329b99c..017258a25a2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -23,8 +23,7 @@ import ca.corefacility.bioinformatics.irida.web.controller.api.samples.RESTSampleSequenceFilesController; import com.google.common.collect.ImmutableMap; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.InputStreamResource; -import org.springframework.core.io.InputStreamSource; +import org.springframework.core.io.ByteArrayResource; import org.springframework.hateoas.Link; import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; @@ -33,6 +32,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +import java.io.IOException; +import java.io.InputStream; import java.util.*; import java.util.stream.Collectors; @@ -267,14 +268,18 @@ public ModelMap getAnalysisOutputFile(@PathVariable Long submissionId, @PathVari * * @param submissionId The {@link AnalysisSubmission} id * @param fileId The {@link AnalysisOutputFile} id - * @return a {@link InputStreamResource} containing the contents of the {@link AnalysisOutputFile}. + * @return a {@link ByteArrayResource} containing the contents of the {@link AnalysisOutputFile}. */ @RequestMapping(value = "/{submissionId}/analysis/file/{fileId}", produces = MediaType.TEXT_PLAIN_VALUE) @ResponseBody - public InputStreamSource getAnalysisOutputFileContents(@PathVariable Long submissionId, - @PathVariable Long fileId) { + public ByteArrayResource getAnalysisOutputFileContents(@PathVariable Long submissionId, + @PathVariable Long fileId) throws IOException { AnalysisOutputFile analysisOutputFile = getOutputFileForSubmission(submissionId, fileId); - return new InputStreamResource(analysisOutputFile.getFileInputStream()); + try(InputStream inputStream = analysisOutputFile.getFileInputStream()) { + return new ByteArrayResource(inputStream.readAllBytes()); + } catch (IOException e) { + throw new IOException("Unable to read input stream ", e); + } } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java index 2c4d001455f..1e0e1fd8fea 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; @@ -47,11 +48,14 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - OutputStream os = response.getOutputStream(); - InputStream is = sfr.getFileInputStream(); - IOUtils.copy(is, os); - is.close(); - os.flush(); - os.close(); + + try(InputStream is = sfr.getFileInputStream()) { + OutputStream os = response.getOutputStream(); + IOUtils.copy(is, os); + os.flush(); + os.close(); + }catch (IOException e) { + throw new IOException("Unable to read inputstream ", e); + } } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java index 48e1a3de969..676fa1d7a91 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java @@ -13,6 +13,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; @@ -46,11 +47,14 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(IridaFiles.getFileSizeBytes(fileContent))); - OutputStream os = response.getOutputStream(); - InputStream is = IridaFiles.getFileInputStream(fileContent); - IOUtils.copy(is, os); - is.close(); - os.flush(); - os.close(); + + try(InputStream is = IridaFiles.getFileInputStream(fileContent)) { + OutputStream os = response.getOutputStream(); + IOUtils.copy(is, os); + os.flush(); + os.close(); + }catch (IOException e) { + throw new IOException("Unable to read inputstream ", e); + } } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java index fdc4782481b..0f94853ce1b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; @@ -45,11 +46,14 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - OutputStream os = response.getOutputStream(); - InputStream is = sfr.getFileInputStream(); - IOUtils.copy(is, os); - is.close(); - os.flush(); - os.close(); + + try(InputStream is = sfr.getFileInputStream()) { + OutputStream os = response.getOutputStream(); + IOUtils.copy(is, os); + os.flush(); + os.close(); + }catch (IOException e) { + throw new IOException("Unable to read inputstream ", e); + } } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java index 87a5233bca1..fffc5a21ec9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; @@ -46,11 +47,14 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - OutputStream os = response.getOutputStream(); - InputStream is = sfr.getFileInputStream(); - IOUtils.copy(is, os); - is.close(); - os.flush(); - os.close(); + + try(InputStream is = sfr.getFileInputStream()) { + OutputStream os = response.getOutputStream(); + IOUtils.copy(is, os); + os.flush(); + os.close(); + }catch (IOException e) { + throw new IOException("Unable to read inputstream ", e); + } } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java index cf54f616c46..a8942a9570c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.web.spring.view; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; @@ -46,11 +47,14 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - OutputStream os = response.getOutputStream(); - InputStream is = sfr.getFileInputStream(); - IOUtils.copy(is, os); - is.close(); - os.flush(); - os.close(); + + try(InputStream is = sfr.getFileInputStream()) { + OutputStream os = response.getOutputStream(); + IOUtils.copy(is, os); + os.flush(); + os.close(); + }catch (IOException e) { + throw new IOException("Unable to read inputstream ", e); + } } } From 8f1edbcf82d40f2a4c5bfeeaa071739a82c1843c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 9 Dec 2020 14:22:23 -0600 Subject: [PATCH 178/655] Added @throws to javadoc --- .../web/controller/api/RESTAnalysisSubmissionController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index 017258a25a2..18f9cf213cf 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -269,6 +269,7 @@ public ModelMap getAnalysisOutputFile(@PathVariable Long submissionId, @PathVari * @param submissionId The {@link AnalysisSubmission} id * @param fileId The {@link AnalysisOutputFile} id * @return a {@link ByteArrayResource} containing the contents of the {@link AnalysisOutputFile}. + * @throws IOException if the file input stream cannot be read */ @RequestMapping(value = "/{submissionId}/analysis/file/{fileId}", produces = MediaType.TEXT_PLAIN_VALUE) @ResponseBody From ad89bd2c21696c879bec53167ab4412a78fa5003 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 9 Dec 2020 16:14:53 -0600 Subject: [PATCH 179/655] Updated to use storageexception rather than illegalstateexception. Updated to use enum comparison. Moved checking if basedirectory exists on the local file system and if it is writable into the iridafilestorageutility --- .../IridaApiFilesystemRepositoryConfig.java | 35 +----------------- .../services/IridaApiServicesConfig.java | 5 ++- .../irida/model/enums/StorageType.java | 16 ++++++++ .../IridaFileStorageAwsUtilityImpl.java | 11 ++++-- .../IridaFileStorageAzureUtilityImpl.java | 9 ++++- .../IridaFileStorageLocalUtilityImpl.java | 37 +++++++++++++++++++ .../filesystem/IridaFileStorageUtility.java | 14 ++++++- 7 files changed, 85 insertions(+), 42 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java index 456956eb92b..e866aad5ed5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/repository/IridaApiFilesystemRepositoryConfig.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.config.repository; import ca.corefacility.bioinformatics.irida.model.assembly.UploadedAssembly; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; @@ -42,9 +41,6 @@ public class IridaApiFilesystemRepositoryConfig { private @Value("${assembly.file.base.directory}") String assemblyFileBaseDirectory; - private @Value("${irida.storage.type}") - String storageType; - @Autowired private ApplicationContext applicationContext; @@ -100,35 +96,8 @@ public Path assemblyFileBaseDirectory() throws IOException { private Path getExistingPathOrThrow(String directory) { Path baseDirectory = Paths.get(directory); - if (!Files.exists(baseDirectory) && storageType.equalsIgnoreCase(StorageType.LOCAL.toString())) { - throw new IllegalStateException( - String.format("Cannot continue startup; base directory [%s] does not exist!", - baseDirectory.toString())); - } else { - if(storageType.equalsIgnoreCase(StorageType.LOCAL.toString())) { - try { - // Check if basedirectory path is writeable by creating a temp file and then removing it - Path tempFile = Files.createTempFile(baseDirectory, "", ""); - // Check if directory is writeable - boolean directoryWriteable = Files.isWritable(tempFile); - - try { - // Cleanup the temp file created in the directory - Files.delete(tempFile); - } catch (IOException e) { - logger.error("An I/O error occurred while attempting to remove temp file ", e); - } - - if(!directoryWriteable) { - // Log the error and exit so startup does not continue - logger.error("Cannot continue startup; base directory " + baseDirectory + " does not have write access! Please check directory permissions."); - System.exit(1); - } - } catch (IOException e) { - logger.error("Unable to create temporary file. Please check directory permissions", e); - System.exit(1); - } - } + boolean baseDirectoryWritable = iridaFileStorageUtility.checkWriteAccess(baseDirectory); + if(baseDirectoryWritable) { logger.info(String.format( "Using specified existing directory at [%s]. The directory *will not* be removed at shutdown time.", baseDirectory.toString())); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 1a1f00dcfd1..0cb898d11b3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -322,10 +322,11 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { @Bean(name = "iridaFileStorageUtility") public IridaFileStorageUtility iridaFileStorageService() { IridaFileStorageUtility iridaFileStorageUtility; - if (storageType.equalsIgnoreCase(StorageType.AWS.toString())) { + StorageType st = StorageType.fromString(storageType); + if (st.equals(StorageType.AWS)) { iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); - } else if (storageType.equalsIgnoreCase(StorageType.AZURE.toString())) { + } else if (st.equals(StorageType.AZURE)) { iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(containerUrl, sasToken, containerName); } else { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java index 520ac9598f6..859e1eeada4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/enums/StorageType.java @@ -23,4 +23,20 @@ private StorageType(String storageType) { public String toString() { return storageType; } + + /** + * Get a storageType from the given storage type + * @param storageType the string to get a storageType for + * @return The requested StorageType + */ + public static StorageType fromString(String storageType) { + switch (storageType.toUpperCase()) { + case "AWS": + return AWS; + case "AZURE": + return AZURE; + default: + return LOCAL; + } + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 354a951501a..8feca55da75 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -310,18 +310,23 @@ public byte[] readAllBytes(Path file) { } @Override - public boolean checkConnectivity() throws IllegalStateException { + public boolean checkConnectivity() throws StorageException { try { if (s3.doesBucketExistV2(bucketName)) { logger.debug("Successfully connected to aws s3 bucket ", bucketName); return true; } else { - throw new IllegalStateException( + throw new StorageException( "Unable to connect to aws s3 bucket. Please check that your credentials are valid and that the bucket " + bucketName + " exists."); } } catch (AmazonServiceException e) { - throw new IllegalStateException( + throw new StorageException( "Unable to connect to aws s3 bucket. Please check that your credentials are valid and that the bucket " + bucketName + " exists."); } } + + @Override + public boolean checkWriteAccess(Path baseDirectory) { + return true; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 5b664182085..b850b15491b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -310,7 +310,7 @@ private String getAzureFileAbsolutePath(Path file) { } @Override - public boolean checkConnectivity() throws IllegalStateException { + public boolean checkConnectivity() throws StorageException { try { // Make a api request to get container properties containerClient.getProperties(); @@ -321,8 +321,13 @@ public boolean checkConnectivity() throws IllegalStateException { Throw an exception which is caught at startup advising the user that the connection was not successful to the azure container */ - throw new IllegalStateException( + throw new StorageException( "Unable to connect to azure container. Please check that your credentials are valid and that the container " + containerName + " exists."); } } + + @Override + public boolean checkWriteAccess(Path baseDirectory) { + return true; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index b35f6b92a43..d7893d5384a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -227,4 +227,41 @@ public boolean checkConnectivity() { return true; } + @Override + public boolean checkWriteAccess(Path baseDirectory) { + if (!Files.exists(baseDirectory)) { + logger.error("Cannot continue startup; base directory " + baseDirectory + " does not exist!"); + System.exit(1); + } else { + try { + // Check if basedirectory path is writeable by creating a temp file and then removing it + Path tempFile = Files.createTempFile(baseDirectory, "", ""); + // Check if directory is writeable + boolean directoryWriteable = Files.isWritable(tempFile); + + try { + // Cleanup the temp file created in the directory + Files.delete(tempFile); + } catch (IOException e) { + logger.error("An I/O error occurred while attempting to remove temp file ", e); + } + + if (!directoryWriteable) { + // Log the error and exit so startup does not continue + logger.error("Cannot continue startup; base directory " + baseDirectory + + " does not have write access! Please check directory permissions."); + System.exit(1); + } + } catch (IOException e) { + logger.error("Unable to create temporary file. Please check directory permissions", e); + System.exit(1); + } + } + /* + If the basedirectory exists and is writeable we return + true otherwise the system will have exited startup + */ + return true; + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 7a22dec7ba9..5f0b37ea358 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.util.List; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; @@ -130,7 +131,16 @@ public interface IridaFileStorageUtility { * api call to check if containers/buckets exist. * * @return Connection valid or not - * @throws IllegalStateException if connection failed + * @throws StorageException if connection failed */ - public boolean checkConnectivity() throws IllegalStateException; + public boolean checkConnectivity() throws StorageException; + + + /** + * Check if the given directory is writable + * + * @param baseDirectory The directory to check write access for + * @return if the directory is writable or not + */ + public boolean checkWriteAccess(Path baseDirectory); } From 660e094f2fa6031b76103846c1ed177080af446a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 10 Dec 2020 09:12:48 -0600 Subject: [PATCH 180/655] Changed illegalstateexception to storageexception as thats what the iridafilestorageutility throws if a connection is not made to a bucket/container --- .../irida/config/services/IridaApiServicesConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 0cb898d11b3..83f286ec01d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -8,6 +8,7 @@ import ca.corefacility.bioinformatics.irida.config.services.conditions.NreplServerSpringCondition; import ca.corefacility.bioinformatics.irida.config.services.scheduled.IridaScheduledTasksConfig; import ca.corefacility.bioinformatics.irida.config.workflow.IridaWorkflowsConfig; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; @@ -335,7 +336,7 @@ public IridaFileStorageUtility iridaFileStorageService() { // Check if connection is valid to local file system or cloud provider try { iridaFileStorageUtility.checkConnectivity(); - } catch (IllegalStateException e) { + } catch (StorageException e) { // Log the error and exit so startup does not continue logger.error("Unable to start up IRIDA! ", e); System.exit(1); From e61f9320a65fa6b71803b841ee38cab2370d1d64 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 10 Dec 2020 11:16:09 -0600 Subject: [PATCH 181/655] Added bean for storageType so we can use it wherever a storage type check is required. Updated azure file storage utility class to check if we can list blobs in a container instead of getting properties as the token is scoped --- .../irida/config/services/IridaApiServicesConfig.java | 5 +++++ .../filesystem/IridaFileStorageAzureUtilityImpl.java | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 83f286ec01d..d182d6695f8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -345,6 +345,11 @@ public IridaFileStorageUtility iridaFileStorageService() { return iridaFileStorageUtility; } + @Bean(name= "storageType") + public StorageType storageType() { + return StorageType.fromString(storageType); + } + @Bean(name = "uploadFileProcessingChain") public FileProcessingChain fileProcessorChain(SequencingObjectRepository sequencingObjectRepository, QCEntryRepository qcRepository, IridaFileStorageUtility iridaFileStorageUtility, GzipFileProcessor gzipFileProcessor, diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index b850b15491b..e98b4c8af20 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -312,11 +312,13 @@ private String getAzureFileAbsolutePath(Path file) { @Override public boolean checkConnectivity() throws StorageException { try { - // Make a api request to get container properties - containerClient.getProperties(); + // Make an api request to check if we can list the blobs in the container + containerClient.listBlobs().forEach(blob -> + blob.getName()); logger.debug("Successfully connected to azure container ", containerName); return true; - } catch (Exception e) { + + } catch (BlobStorageException e) { /* Throw an exception which is caught at startup advising the user that the connection was not successful to the azure container From 97d24e6e32d561b8e78c9cef3a8161074168d7eb Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 10 Dec 2020 13:10:46 -0600 Subject: [PATCH 182/655] Removed class variable not required. Updated logging statements --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 2 +- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 8feca55da75..abd461b3b73 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -313,7 +313,7 @@ public byte[] readAllBytes(Path file) { public boolean checkConnectivity() throws StorageException { try { if (s3.doesBucketExistV2(bucketName)) { - logger.debug("Successfully connected to aws s3 bucket ", bucketName); + logger.debug("Successfully connected to aws s3 bucket " + bucketName); return true; } else { throw new StorageException( diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index e98b4c8af20..7456c2e8d5c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -38,15 +38,12 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; - private String containerName; - @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl) .sasToken(sasToken) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - this.containerName = containerName; } /** @@ -311,11 +308,12 @@ private String getAzureFileAbsolutePath(Path file) { @Override public boolean checkConnectivity() throws StorageException { + String containerName = containerClient.getBlobContainerName(); try { // Make an api request to check if we can list the blobs in the container containerClient.listBlobs().forEach(blob -> blob.getName()); - logger.debug("Successfully connected to azure container ", containerName); + logger.debug("Successfully connected to azure container " + containerName); return true; } catch (BlobStorageException e) { From c73473d1a625637a6d1a571726bfcc3f7faa311e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 10 Dec 2020 15:40:07 -0600 Subject: [PATCH 183/655] Updated file models to getStorageType from IridaFiles which is set in the rest api response for files --- .../bioinformatics/irida/model/assembly/GenomeAssembly.java | 4 ++++ .../irida/model/sequenceFile/SequenceFile.java | 4 ++++ .../filesystem/IridaFileStorageAwsUtilityImpl.java | 5 +++++ .../filesystem/IridaFileStorageAzureUtilityImpl.java | 5 +++++ .../filesystem/IridaFileStorageLocalUtilityImpl.java | 5 +++++ .../repositories/filesystem/IridaFileStorageUtility.java | 1 + .../corefacility/bioinformatics/irida/util/IridaFiles.java | 4 ++++ 7 files changed, 28 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 2b5e3935136..b43502e1463 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -151,4 +151,8 @@ public boolean equals(Object obj) { public InputStream getFileInputStream() { return IridaFiles.getFileInputStream(getFile()); } + + public String getStorageType(){ + return IridaFiles.getStorageType(); + } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 55c21c5210a..5f0905b8663 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -323,4 +323,8 @@ public Long getFileSizeBytes() { return IridaFiles.getFileSizeBytes(getFile()); } + public String getStorageType(){ + return IridaFiles.getStorageType(); + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 30785be5e98..0e30625dae1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -356,4 +356,9 @@ public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { } return null; } + + @Override + public String getStorageType() { + return "aws"; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 388a5e01c57..0b2cea7dd79 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -357,4 +357,9 @@ public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { } return null; } + + @Override + public String getStorageType() { + return "azure"; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 8a65976a1b9..e1d3e7f276b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -253,4 +253,9 @@ public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { } return null; } + + @Override + public String getStorageType() { + return "local"; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index c2274036337..1559039d82f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -143,4 +143,5 @@ public interface IridaFileStorageUtility { */ public FileChunkResponse readChunk(Path file, Long seek, Long chunk); + public String getStorageType(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 17d6843d44e..e395cfd85b0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -96,4 +96,8 @@ public static Long getFileSizeBytes(Path file) { */ public static boolean fileExists(Path file) { return iridaFileStorageUtility.fileExists(file);} + public static String getStorageType() { + return iridaFileStorageUtility.getStorageType(); + } + } From 3b6cbfba6e48e8897b16d0c8dab70e5df0b63827 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 11 Dec 2020 12:31:12 -0600 Subject: [PATCH 184/655] Added missing javadcos --- .../repositories/filesystem/IridaFileStorageUtility.java | 5 +++++ .../corefacility/bioinformatics/irida/util/IridaFiles.java | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 1559039d82f..b325283831a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -143,5 +143,10 @@ public interface IridaFileStorageUtility { */ public FileChunkResponse readChunk(Path file, Long seek, Long chunk); + /** + * Get the storage type. + * + * @return {@link String} The storage type + */ public String getStorageType(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index e395cfd85b0..6f5f9e3b5ff 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -96,8 +96,12 @@ public static Long getFileSizeBytes(Path file) { */ public static boolean fileExists(Path file) { return iridaFileStorageUtility.fileExists(file);} + /** + * Get the storage type from the iridaFileStorageUtility. + * + * @return {@link String} The storage type + */ public static String getStorageType() { return iridaFileStorageUtility.getStorageType(); } - } From 0708c260ae84f48ad42cb53563b35d72d98d2e35 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 16 Dec 2020 15:43:08 -0600 Subject: [PATCH 185/655] Updated to clean up temp files downloaded to server from an object store when running an analysis submission with cloud stored files --- .../AnalysisExecutionServiceConfig.java | 10 +- .../analysis/ExecutionManagerConfig.java | 2 +- .../AnalysisScheduledTaskConfig.java | 6 +- .../AnalysisSubmissionTempFile.java | 96 +++++++++++++++++++ .../upload/galaxy/GalaxyLibrariesService.java | 22 ++--- .../AnalysisSubmissionTempFileRepository.java | 22 +++++ .../AnalysisCollectionServiceGalaxy.java | 71 +++++++++++--- .../AnalysisWorkspaceServiceGalaxy.java | 4 +- .../AnalysisExecutionScheduledTaskImpl.java | 53 +++++++--- .../irida/database/all-changes.xml | 1 + .../database/changesets/21.01/all-changes.xml | 9 ++ .../analysis-submission-cloud-temp-files.xml | 32 +++++++ .../config/IridaApiNoGalaxyTestConfig.java | 2 +- .../AnalysisExecutionServiceTestConfig.java | 10 +- .../analysis/GalaxyExecutionTestConfig.java | 2 +- .../integration/SNVPhylAnalysisIT.java | 6 +- .../integration/GalaxyHistoriesServiceIT.java | 4 +- .../integration/GalaxyJobErrorsServiceIT.java | 6 +- .../integration/GalaxyLibrariesServiceIT.java | 4 +- .../galaxy/integration/GalaxyWorkflowsIT.java | 2 +- .../unit/GalaxyLibrariesServiceTest.java | 14 +-- .../AnalysisCollectionServiceGalaxyIT.java | 12 ++- .../AnalysisWorkspaceServiceGalaxyIT.java | 2 +- .../AnalysisWorkspaceServiceGalaxyTest.java | 20 ++-- .../AnalysisExecutionScheduledTaskImplIT.java | 8 +- ...nalysisExecutionScheduledTaskImplTest.java | 13 ++- 26 files changed, 348 insertions(+), 85 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java index cd88ac1aabf..edad2b0f94d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java @@ -25,6 +25,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; import ca.corefacility.bioinformatics.irida.plugins.IridaPlugin; import ca.corefacility.bioinformatics.irida.plugins.IridaPluginException; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; import ca.corefacility.bioinformatics.irida.service.AnalysisService; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; @@ -105,6 +107,12 @@ public class AnalysisExecutionServiceConfig { @Autowired private List defaultAnalysisSampleUpdaters; + @Autowired + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; + private List loadPluginAnalysisSampleUpdaters() { List pluginUpdaters = Lists.newLinkedList(); @@ -171,6 +179,6 @@ public AnalysisProvenanceServiceGalaxy analysisProvenanceService() { @Lazy @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { - return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService); + return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java index 0199611cf32..29ea949f0c5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java @@ -289,7 +289,7 @@ public GalaxyHistoriesService galaxyHistoriesService() throws ExecutionManagerCo @Lazy @Bean public GalaxyLibrariesService galaxyLibrariesService() throws ExecutionManagerConfigurationException { - return new GalaxyLibrariesService(librariesClient(), pollingTime, libraryTimeout, libraryUploadThreads, iridaFileStorageUtility); + return new GalaxyLibrariesService(librariesClient(), pollingTime, libraryTimeout, libraryUploadThreads); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java index fa44838fb05..3e1abb7ed83 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java @@ -2,6 +2,7 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; @@ -54,6 +55,9 @@ public class AnalysisScheduledTaskConfig { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; + @Autowired + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + /** * Defines the time to clean up in number of days a submission must exist before it is cleaned up. */ @@ -129,7 +133,7 @@ public void cleanupAnalysisSubmissions() { public AnalysisExecutionScheduledTask analysisExecutionScheduledTask() { return new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, cleanupAnalysisSubmissionCondition(), galaxyJobErrorsService, jobErrorRepository, emailController, - analysisWorkspaceService, iridaFileStorageUtility); + analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java new file mode 100644 index 00000000000..c111739cd1f --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java @@ -0,0 +1,96 @@ +package ca.corefacility.bioinformatics.irida.model.workflow.submission; + +import java.nio.file.Path; +import java.util.Date; + +import javax.persistence.*; +import javax.validation.constraints.NotNull; + +import ca.corefacility.bioinformatics.irida.model.IridaThing; +import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; + +/** + * A temporary file which required by an {@link AnalysisSubmission} when + * the storage type is an object store. + */ +@Entity +@Table(name = "analysis_submission_temp_files") +public class AnalysisSubmissionTempFile implements IridaThing { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private final Long id; + + @NotNull + @Column(name = "analysis_submission_id") + private final Long analysisSubmissionId; + + @NotNull + @Column(name = "temp_file_path") + private final Path filePath; + + @NotNull + @Column(name = "temp_file_directory_path") + private final Path fileDirectoryPath; + + @NotNull + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "created_date") + private final Date createdDate; + + /** + * for hibernate + */ + @SuppressWarnings("unused") + private AnalysisSubmissionTempFile() { + this.analysisSubmissionId = null; + this.filePath = null; + this.fileDirectoryPath = null; + this.id = null; + this.createdDate = null; + } + + /** + * Create a new {@link AnalysisSubmissionTempFile} with the given file + * analysis submission id, file path, and directory path. + * + * @param analysisSubmissionId The id of the {@link AnalysisSubmission} + * @param filePath The path to the temporary file + * @param fileDirectoryPath The path to the temporary file directory + */ + public AnalysisSubmissionTempFile(Long analysisSubmissionId, Path filePath, Path fileDirectoryPath) { + this.analysisSubmissionId = analysisSubmissionId; + this.filePath = filePath; + this.fileDirectoryPath = fileDirectoryPath; + this.id = null; + this.createdDate = new Date(); + } + + /** + * Get the implementation-specific file label. + * + * @return the file label. + */ + @Override + public String getLabel() { + return filePath.getFileName().toString(); + } + + @Override + public Date getCreatedDate() { + return this.createdDate; + } + + @Override + public Long getId() { + return this.id; + } + + public Path getFilePath() { + return filePath; + } + + public Path getFileDirectoryPath() { + return fileDirectoryPath; + } + +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index 59e4b51d87a..78c35949108 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -30,14 +30,10 @@ import ca.corefacility.bioinformatics.irida.model.upload.galaxy.GalaxyProjectName; import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.github.jmchilton.blend4j.galaxy.LibrariesClient; -import com.github.jmchilton.blend4j.galaxy.beans.FilesystemPathsLibraryUpload; -import com.github.jmchilton.blend4j.galaxy.beans.GalaxyObject; -import com.github.jmchilton.blend4j.galaxy.beans.Library; -import com.github.jmchilton.blend4j.galaxy.beans.LibraryContent; -import com.github.jmchilton.blend4j.galaxy.beans.LibraryDataset; +import com.github.jmchilton.blend4j.galaxy.beans.*; import com.google.common.collect.Lists; import com.sun.jersey.api.client.ClientResponse; @@ -58,7 +54,6 @@ public class GalaxyLibrariesService { private final int libraryPollingTime; private final int libraryUploadTimeout; - private IridaFileStorageUtility iridaFileStorageUtility; /** * State a library dataset should be in on proper upload. @@ -79,10 +74,9 @@ public class GalaxyLibrariesService { * @param libraryUploadTimeout The timeout (in seconds) for waiting for files to be uploaded * to a library. * @param threadPoolSize The thread pool size for parallel polling of Galaxy to check if uploads are finished. - * @param iridaFileStorageUtility The file storage implementation */ public GalaxyLibrariesService(LibrariesClient librariesClient, final int libraryPollingTime, - final int libraryUploadTimeout, final int threadPoolSize, IridaFileStorageUtility iridaFileStorageUtility) { + final int libraryUploadTimeout, final int threadPoolSize) { checkNotNull(librariesClient, "librariesClient is null"); checkArgument(libraryPollingTime > 0, "libraryPollingTime=" + libraryPollingTime + " must be positive"); checkArgument(libraryUploadTimeout > 0, "libraryUploadTimeout=" + libraryUploadTimeout + " must be positive"); @@ -98,7 +92,6 @@ public GalaxyLibrariesService(LibrariesClient librariesClient, final int library this.librariesClient = librariesClient; this.libraryPollingTime = libraryPollingTime; this.libraryUploadTimeout = libraryUploadTimeout; - this.iridaFileStorageUtility = iridaFileStorageUtility; executor = Executors.newFixedThreadPool(threadPoolSize); } @@ -153,9 +146,9 @@ public String fileToLibrary(Path path, InputFileType fileType, checkNotNull(fileType, "fileType is null"); checkNotNull(library, "library is null"); checkNotNull(library.getId(), "library id is null"); - checkState(iridaFileStorageUtility.fileExists(path), "path " + path + " does not exist"); + checkState(path.toFile().exists(), "path " + path + " does not exist"); - File file = iridaFileStorageUtility.getTemporaryFile(path).getFile().toFile(); + File file = path.toFile(); try { LibraryContent rootContent = librariesClient.getRootFolder(library @@ -171,6 +164,7 @@ public String fileToLibrary(Path path, InputFileType fileType, GalaxyObject uploadObject = librariesClient.uploadFilesystemPaths( library.getId(), upload); + return uploadObject.getId(); } catch (RuntimeException e) { throw new UploadException(e); @@ -265,9 +259,9 @@ public Void call() throws Exception { * @throws IOException If there was an error reading the file to determine the file type. */ private InputFileType getFileType(Path path) throws IOException { - checkArgument(iridaFileStorageUtility.fileExists(path), "path=[" + path + "] does not exist"); + checkArgument(path.toFile().exists(), "path=[" + path + "] does not exist"); - return (iridaFileStorageUtility.isGzipped(path) ? InputFileType.FASTQ_SANGER_GZ : InputFileType.FASTQ_SANGER); + return (FileUtils.isGzipped(path) ? InputFileType.FASTQ_SANGER_GZ : InputFileType.FASTQ_SANGER); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java new file mode 100644 index 00000000000..d30d82fba4a --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java @@ -0,0 +1,22 @@ +package ca.corefacility.bioinformatics.irida.repositories.analysis.submission; + +import java.util.List; + + +import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; +import ca.corefacility.bioinformatics.irida.repositories.IridaJpaRepository; + +/** + * A repository for managing {@link AnalysisSubmissionTempFile} objects. + */ + +public interface AnalysisSubmissionTempFileRepository extends IridaJpaRepository { + + /** + * Get all {@link AnalysisSubmissionTempFile} objects by submission id. + * + * @param analysisSubmissionId The analysis submission id + * @return a list of {@link AnalysisSubmissionTempFile} + */ + List findAllByAnalysisSubmissionId(Long analysisSubmissionId); +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index 5c18cafb88d..29ffff713c2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -7,8 +7,14 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.DatasetCollectionType; +import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; +import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; + import com.github.jmchilton.blend4j.galaxy.beans.History; import com.github.jmchilton.blend4j.galaxy.beans.Library; import com.github.jmchilton.blend4j.galaxy.beans.collection.request.CollectionDescription; @@ -37,6 +43,8 @@ public class AnalysisCollectionServiceGalaxy { private static final String REVERSE_NAME = "reverse"; private GalaxyHistoriesService galaxyHistoriesService; + private IridaFileStorageUtility iridaFileStorageUtility; + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; /** * Builds a new {@link AnalysisCollectionServiceGalaxy} with the given @@ -45,26 +53,29 @@ public class AnalysisCollectionServiceGalaxy { * @param galaxyHistoriesService A GalaxyHistoriesService for interacting with * Galaxy Histories. */ - public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService) { + public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, IridaFileStorageUtility iridaFileStorageUtility, AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { this.galaxyHistoriesService = galaxyHistoriesService; + this.iridaFileStorageUtility = iridaFileStorageUtility; + this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; } /** * Uploads a list of single sequence files belonging to the given samples to * Galaxy. - * + * * @param sampleSequenceFiles A map between {@link Sample} and * {@link SingleEndSequenceFile}. * @param workflowHistory The history to upload the sequence files into. * @param workflowLibrary A temporary library to upload files into. + * @param submissionId The {@link AnalysisSubmission} id * @return A CollectionResponse for the dataset collection constructed from the - * given files. + * given files. * @throws ExecutionManagerException If there was an error uploading the files. - * @throws IOException If there was an error reading the sequence file. + * @throws IOException If there was an error reading the sequence file. */ public CollectionResponse uploadSequenceFilesSingleEnd( Map sampleSequenceFiles, History workflowHistory, - Library workflowLibrary) throws ExecutionManagerException, IOException { + Library workflowLibrary, Long submissionId) throws ExecutionManagerException, IOException { CollectionDescription description = new CollectionDescription(); description.setCollectionType(DatasetCollectionType.LIST.toString()); @@ -73,7 +84,19 @@ public CollectionResponse uploadSequenceFilesSingleEnd( Map samplesMap = new HashMap<>(); for (Sample sample : sampleSequenceFiles.keySet()) { SingleEndSequenceFile sequenceFile = sampleSequenceFiles.get(sample); - samplesMap.put(sequenceFile.getSequenceFile().getFile(), sample); + IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile()); + samplesMap.put(iridaTemporaryFile.getFile(), sample); + + /* + If the storage type is an object store then we create an AnalysisSubmissionTempFile + record to save to the database. The cleanup of these files happens once the final + workflow status is set for an analysis. + */ + if (!iridaFileStorageUtility.storageTypeIsLocal()) { + AnalysisSubmissionTempFile analysisSubmissionTempFile = new AnalysisSubmissionTempFile(submissionId, + iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); + analysisSubmissionTempFileRepository.save(analysisSubmissionTempFile); + } } // upload files to library and then to a history @@ -101,20 +124,21 @@ public CollectionResponse uploadSequenceFilesSingleEnd( /** * Uploads a list of paired sequence files belonging to the given samples to * Galaxy. - * + * * @param sampleSequenceFilesPaired A map between {@link Sample} and * {@link SequenceFilePair}. * @param workflowHistory The history to upload the sequence files * into. * @param workflowLibrary A temporary library to upload files into. + * @param submissionId The {@link AnalysisSubmission} id * @return A CollectionResponse for the dataset collection constructed from the - * given files. + * given files. * @throws ExecutionManagerException If there was an error uploading the files. - * @throws IOException If there was an error reading the sequence file. + * @throws IOException If there was an error reading the sequence file. */ public CollectionResponse uploadSequenceFilesPaired( Map sampleSequenceFilesPaired, History workflowHistory, - Library workflowLibrary) throws ExecutionManagerException, IOException { + Library workflowLibrary, Long submissionId) throws ExecutionManagerException, IOException { CollectionDescription description = new CollectionDescription(); description.setCollectionType(DatasetCollectionType.LIST_PAIRED.toString()); @@ -128,10 +152,29 @@ public CollectionResponse uploadSequenceFilesPaired( SequenceFile fileForward = sequenceFilePair.getForwardSequenceFile(); SequenceFile fileReverse = sequenceFilePair.getReverseSequenceFile(); - samplesMapPairForward.put(sample, fileForward.getFile()); - samplesMapPairReverse.put(sample, fileReverse.getFile()); - pathsToUpload.add(fileForward.getFile()); - pathsToUpload.add(fileReverse.getFile()); + IridaTemporaryFile iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile()); + IridaTemporaryFile iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile()); + + samplesMapPairForward.put(sample, iridaTemporaryFileForward.getFile()); + samplesMapPairReverse.put(sample, iridaTemporaryFileReverse.getFile()); + pathsToUpload.add(iridaTemporaryFileForward.getFile()); + pathsToUpload.add(iridaTemporaryFileReverse.getFile()); + + /* + If the storage type is an object store then we create an AnalysisSubmissionTempFile + record to save to the database. The cleanup of these files happens once the final + workflow status is set for an analysis. + */ + if (!iridaFileStorageUtility.storageTypeIsLocal()) { + AnalysisSubmissionTempFile analysisSubmissionTempFileForward = new AnalysisSubmissionTempFile( + submissionId, iridaTemporaryFileForward.getFile(), + iridaTemporaryFileForward.getDirectoryPath()); + AnalysisSubmissionTempFile analysisSubmissionTempFileReverse = new AnalysisSubmissionTempFile( + submissionId, iridaTemporaryFileReverse.getFile(), + iridaTemporaryFileReverse.getDirectoryPath()); + analysisSubmissionTempFileRepository.save(analysisSubmissionTempFileForward); + analysisSubmissionTempFileRepository.save(analysisSubmissionTempFileReverse); + } } // upload files to library and then to a history diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java index e1e24169088..18f1bd91a6b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java @@ -220,7 +220,7 @@ public PreparedWorkflowGalaxy prepareAnalysisFiles(AnalysisSubmission analysisSu String workflowSequenceFileSingleInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, sequenceFilesLabelSingle); CollectionResponse collectionResponseSingle = analysisCollectionServiceGalaxy - .uploadSequenceFilesSingleEnd(singleFiles, workflowHistory, workflowLibrary); + .uploadSequenceFilesSingleEnd(singleFiles, workflowHistory, workflowLibrary, analysisSubmission.getId()); inputs.setInput(workflowSequenceFileSingleInputId, new WorkflowInvocationInputs.WorkflowInvocationInput( collectionResponseSingle.getId(), WorkflowInvocationInputs.InputSourceType.HDCA)); } @@ -230,7 +230,7 @@ public PreparedWorkflowGalaxy prepareAnalysisFiles(AnalysisSubmission analysisSu String workflowSequenceFilePairedInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, sequenceFilesLabelPaired); CollectionResponse collectionResponsePaired = analysisCollectionServiceGalaxy - .uploadSequenceFilesPaired(pairedFiles, workflowHistory, workflowLibrary); + .uploadSequenceFilesPaired(pairedFiles, workflowHistory, workflowLibrary, analysisSubmission.getId()); inputs.setInput(workflowSequenceFilePairedInputId, new WorkflowInvocationInputs.WorkflowInvocationInput( collectionResponsePaired.getId(), WorkflowInvocationInputs.InputSourceType.HDCA)); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index 6e5d92e8a07..d96783bae54 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -19,10 +19,13 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; @@ -56,26 +59,29 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche private final EmailController emailController; private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisWorkspaceService analysisWorkspaceService; + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; /** * Builds a new AnalysisExecutionScheduledTaskImpl with the given service * classes. * - * @param analysisSubmissionRepository A repository for {@link AnalysisSubmission}s. - * @param analysisExecutionServiceGalaxy A service for executing {@link AnalysisSubmission}s. - * @param cleanupCondition The condition defining when an {@link AnalysisSubmission} - * should be cleaned up. - * @param galaxyJobErrorsService {@link GalaxyJobErrorsService} for getting {@link JobError} objects - * @param jobErrorRepository {@link JobErrorRepository} for {@link JobError} objects - * @param emailController {@link EmailController} for sending completion/error emails for {@link AnalysisSubmission}s - * @param analysisWorkspaceService {@link AnalysisWorkspaceService} - * @param iridaFileStorageUtility The irida file storage utility implementation + * @param analysisSubmissionRepository A repository for {@link AnalysisSubmission}s. + * @param analysisExecutionServiceGalaxy A service for executing {@link AnalysisSubmission}s. + * @param cleanupCondition The condition defining when an {@link AnalysisSubmission} + * should be cleaned up. + * @param galaxyJobErrorsService {@link GalaxyJobErrorsService} for getting {@link JobError} objects + * @param jobErrorRepository {@link JobErrorRepository} for {@link JobError} objects + * @param emailController {@link EmailController} for sending completion/error emails for {@link AnalysisSubmission}s + * @param analysisWorkspaceService {@link AnalysisWorkspaceService} + * @param iridaFileStorageUtility The irida file storage utility implementation + * @param analysisSubmissionTempFileRepository {@link AnalysisSubmissionTempFileRepository} for {@link AnalysisSubmissionTempFile} objects */ @Autowired public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisSubmissionRepository, AnalysisExecutionService analysisExecutionServiceGalaxy, CleanupAnalysisSubmissionCondition cleanupCondition, GalaxyJobErrorsService galaxyJobErrorsService, - JobErrorRepository jobErrorRepository, EmailController emailController, AnalysisWorkspaceService analysisWorkspaceService, IridaFileStorageUtility iridaFileStorageUtility) { + JobErrorRepository jobErrorRepository, EmailController emailController, AnalysisWorkspaceService analysisWorkspaceService, IridaFileStorageUtility iridaFileStorageUtility, + AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { this.analysisSubmissionRepository = analysisSubmissionRepository; this.analysisExecutionService = analysisExecutionServiceGalaxy; this.cleanupCondition = cleanupCondition; @@ -84,6 +90,7 @@ public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisS this.emailController = emailController; this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisWorkspaceService = analysisWorkspaceService; + this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; } /** @@ -306,11 +313,31 @@ private Future handleWorkflowStatus(GalaxyWorkflowStatus wor /* The variable finalWorkflowStatusSet is set to true when an analysis has successfully completed or completed with an error and is used in - the logic below. If the analysis has finished with an error or completed successfully - and the user selected to be emailed on completion, then the following code - will be executed. + the logic below. */ if (finalWorkflowStatusSet) { + + /* + Cleanup any files that were downloaded from an object store to run an analysis and + remove the analysis submission temp file record from the database. + */ + if (!iridaFileStorageUtility.storageTypeIsLocal()) { + List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( + analysisSubmission.getId()); + logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + " temporary files downloaded from object store."); + for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( + new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), + analysisSubmissionTempFile.getFileDirectoryPath())); + analysisSubmissionTempFileRepository.delete(analysisSubmissionTempFile); + } + } + + /* + If the analysis has finished with an error or completed successfully + and the user selected to be emailed on completion, then the following code + will be executed. + */ if(analysisSubmission.getEmailPipelineResult()) { emailController.sendPipelineStatusEmail(analysisSubmission); } diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml index d9cd59dab6a..c7667aa7893 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml @@ -69,4 +69,5 @@ + diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml new file mode 100644 index 00000000000..d23a2316b0e --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml @@ -0,0 +1,9 @@ + + + + + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml new file mode 100644 index 00000000000..6f79646045c --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java index 9d8598870ec..4e4ca7c0abe 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java @@ -98,7 +98,7 @@ public GalaxyHistoriesService galaxyHistoriesService(HistoriesClient historiesCl @Bean public GalaxyLibrariesService galaxyLibrariesService(LibrariesClient librariesClient) { - return new GalaxyLibrariesService(librariesClient, 5, 60, 1, iridaFileStorageUtility); + return new GalaxyLibrariesService(librariesClient, 5, 60, 1); } @Bean diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java index aa3af4a7d91..ac03cba8a30 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java @@ -33,6 +33,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.referencefile.ReferenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; import ca.corefacility.bioinformatics.irida.service.AnalysisService; @@ -102,6 +104,12 @@ public class AnalysisExecutionServiceTestConfig { @Autowired private SampleRepository sampleRepository; + + @Autowired + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + + @Autowired + private IridaFileStorageUtility iridaFileStorageUtility; @Bean public AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor() { @@ -143,7 +151,7 @@ galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy() @Lazy @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { - return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService); + return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java index b13644c27bb..bdb23329542 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java @@ -62,7 +62,7 @@ public GalaxyHistoriesService galaxyHistoriesService() { @Bean public GalaxyLibrariesService galaxyLibrariesService() { LibrariesClient librariesClient = localGalaxy.getGalaxyInstanceAdmin().getLibrariesClient(); - return new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageUtility); + return new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java index 11515e22141..9d13b66e827 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java @@ -58,6 +58,7 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; @@ -133,6 +134,9 @@ public class SNVPhylAnalysisIT { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; + @Autowired + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + private Path sequenceFilePathA1; private Path sequenceFilePathA2; @@ -179,7 +183,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.NEVER_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); Path tempDir = Files.createTempDirectory(rootTempDirectory, "snvphylTest"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java index e5c958e0a58..72e086eb282 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java @@ -106,6 +106,8 @@ public class GalaxyHistoriesServiceIT { */ private static final int LIBRARY_POLLING_TIME = 5; + private static final Long ANALYSIS_SUBMISSION_ID = 1L; + /** * Sets up files for history tests. * @throws URISyntaxException @@ -123,7 +125,7 @@ public void setup() throws URISyntaxException, IOException, CreateLibraryExcepti LibrariesClient librariesClient = galaxyInstanceAdmin.getLibrariesClient(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageUtility); + galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageUtility); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java index c2e0f193165..85f36726add 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java @@ -29,6 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.EmailController; @@ -87,6 +88,9 @@ public class GalaxyJobErrorsServiceIT { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; + @Autowired + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + @Before public void setup() throws URISyntaxException, IOException { Assume.assumeFalse(WindowsPlatformCondition.isWindows()); @@ -105,7 +109,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java index 187ddfb9ef3..2861c0d0ea4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyLibrariesServiceIT.java @@ -91,7 +91,7 @@ public void setup() throws URISyntaxException, IOException { librariesClient = galaxyInstanceAdmin.getLibrariesClient(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageUtility); + galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); dataFile = Paths.get(GalaxyLibrariesServiceIT.class.getResource( "testData1.fastq").toURI()); @@ -211,7 +211,7 @@ public void testFilesToLibraryWaitFail() */ @Test(expected = UploadTimeoutException.class) public void testFilesToLibraryWaitFailTimeout() throws UploadException, GalaxyDatasetException { - galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageUtility); + galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, 1, 2, 1); Library library = buildEmptyLibrary("testFilesToLibraryWaitFailTimeout"); galaxyLibrariesService.filesToLibraryWait(Sets.newHashSet(dataFile, dataFile2), library, diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java index 4f3a21715e0..7dea6716d0d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java @@ -158,7 +158,7 @@ public void setup() throws URISyntaxException, IOException { librariesClient = galaxyAdminInstance.getLibrariesClient(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageUtility); + GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageUtility); galaxyWorkflowService = new GalaxyWorkflowService(workflowsClient, StandardCharsets.UTF_8); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java index b722b740daf..190edb82d94 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyLibrariesServiceTest.java @@ -59,7 +59,7 @@ private void setupLibrariesTest() { */ @Test(expected=IllegalArgumentException.class) public void testZeroPollingTime() { - new GalaxyLibrariesService(librariesClient, 0, 1, 1, iridaFileStorageUtility); + new GalaxyLibrariesService(librariesClient, 0, 1, 1); } /** @@ -67,7 +67,7 @@ public void testZeroPollingTime() { */ @Test(expected=IllegalArgumentException.class) public void testZeroUploadTimeout() { - new GalaxyLibrariesService(librariesClient, 1, 0, 1, iridaFileStorageUtility); + new GalaxyLibrariesService(librariesClient, 1, 0, 1); } /** @@ -75,7 +75,7 @@ public void testZeroUploadTimeout() { */ @Test(expected=IllegalArgumentException.class) public void testEqualPollingTimeUploadTimeout() { - new GalaxyLibrariesService(librariesClient, 1, 1, 1, iridaFileStorageUtility); + new GalaxyLibrariesService(librariesClient, 1, 1, 1); } /** @@ -83,7 +83,7 @@ public void testEqualPollingTimeUploadTimeout() { */ @Test public void testSuccessfullTimeoutValues() { - new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageUtility); + new GalaxyLibrariesService(librariesClient, 1, 2, 1); } /** @@ -91,7 +91,7 @@ public void testSuccessfullTimeoutValues() { */ @Test(expected=IllegalArgumentException.class) public void testFailThreadValue() { - new GalaxyLibrariesService(librariesClient, 1, 2, 0, iridaFileStorageUtility); + new GalaxyLibrariesService(librariesClient, 1, 2, 0); } /** @@ -103,7 +103,7 @@ public void testBuildEmptyLibrary() throws CreateLibraryException { when(librariesClient.createLibrary(any(Library.class))).thenReturn( testLibrary); - Library library = new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageUtility).buildEmptyLibrary(new GalaxyProjectName( + Library library = new GalaxyLibrariesService(librariesClient, 1, 2, 1).buildEmptyLibrary(new GalaxyProjectName( "test")); assertNotNull(library); @@ -120,6 +120,6 @@ public void testBuildEmptyLibrary() throws CreateLibraryException { public void testBuildEmptyLibraryFail() throws CreateLibraryException { when(librariesClient.createLibrary(any(Library.class))).thenReturn(null); - new GalaxyLibrariesService(librariesClient, 1, 2, 1, iridaFileStorageUtility).buildEmptyLibrary(new GalaxyProjectName("test")); + new GalaxyLibrariesService(librariesClient, 1, 2, 1).buildEmptyLibrary(new GalaxyProjectName("test")); } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java index 7360aa43408..c96114214bb 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java @@ -121,6 +121,8 @@ public class AnalysisCollectionServiceGalaxyIT { private static final String FORWARD_NAME = "forward"; private static final String REVERSE_NAME = "reverse"; + private static final Long ANALYSIS_SUBMISSION_ID = 1L; + /** * Sets up variables for testing. * @@ -284,7 +286,7 @@ public void testUploadSequenceFilesSingleSuccess() throws ExecutionManagerExcept Sample sample1 = sampleRepository.findById(1L).orElse(null); CollectionResponse collectionResponse = analysisCollectionServiceGalaxy - .uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, createdLibrary); + .uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, createdLibrary, ANALYSIS_SUBMISSION_ID); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -348,7 +350,7 @@ public void testUploadSequenceFilesSingleCompressedSuccess() throws ExecutionMan sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, - createdLibrary); + createdLibrary, ANALYSIS_SUBMISSION_ID); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -396,7 +398,7 @@ public void testUploadSequenceFilesPairedSuccess() throws ExecutionManagerExcept Sample sample1 = sampleRepository.findById(1L).orElse(null); CollectionResponse collectionResponse = analysisCollectionServiceGalaxy - .uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, createdLibrary); + .uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, createdLibrary, ANALYSIS_SUBMISSION_ID); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -495,7 +497,7 @@ public void testUploadSequenceFilesPairedCompressedSuccess() throws ExecutionMan sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, - createdLibrary); + createdLibrary, ANALYSIS_SUBMISSION_ID); SequenceFilePair pairedSequenceFile = sequenceFiles.iterator().next(); for (SequenceFile file : pairedSequenceFile.getFiles()) { @@ -551,7 +553,7 @@ public void testUploadSequenceFilesPairedFailForward() throws ExecutionManagerEx Map sampleSequenceFilePairs = new HashMap<>(sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, - createdLibrary); + createdLibrary, ANALYSIS_SUBMISSION_ID); } private Map historyContentsAsMap(List historyContents) { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java index fcf0a7b0993..90adc379f55 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java @@ -233,7 +233,7 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc LibrariesClient librariesClient = galaxyInstanceAdmin.getLibrariesClient(); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1, iridaFileStorageUtility); + GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); galaxyHistoriesService = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageUtility); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index 6c0cfdaed42..384e9ba978e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -298,9 +298,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary))).thenReturn(collectionResponseSingle); + eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponseSingle); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary))).thenReturn(collectionResponsePaired); + eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -318,9 +318,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class)); + any(Library.class), any(Long.class)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class)); + any(Library.class), any(Long.class)); } /** @@ -363,7 +363,7 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary))).thenReturn(collectionResponseSingle); + eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponseSingle); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -380,9 +380,9 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file single entry", workflowInputsMap.containsKey(SEQUENCE_FILE_SINGLE_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class)); + any(Library.class), any(Long.class)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class)); + any(Library.class), any(Long.class)); } /** @@ -426,7 +426,7 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary))).thenReturn(collectionResponsePaired); + eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -443,9 +443,9 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesSingleEnd(any(Map.class), - any(History.class), any(Library.class)); + any(History.class), any(Library.class), any(Long.class)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class)); + any(Library.class), any(Long.class)); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java index de1bf7cec6b..f32c654b848 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java @@ -40,6 +40,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.user.UserRepository; @@ -101,6 +102,9 @@ public class AnalysisExecutionScheduledTaskImplIT { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; + @Autowired + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + private Path sequenceFilePath; private Path sequenceFilePath2; @@ -125,7 +129,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); Path sequenceFilePathReal = Paths .get(DatabaseSetupGalaxyITService.class.getResource("testData1.fastq").toURI()); @@ -192,7 +196,7 @@ public void testFullAnalysisRunSuccessWithSampleUpdates() throws Exception { public void testFullAnalysisRunSuccessNoCleanupAge() throws Exception { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); AnalysisSubmission analysisSubmission = analysisExecutionGalaxyITService.setupSubmissionInDatabase(1L, sequenceFilePath, referenceFilePath, validIridaWorkflowId, false); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index b1f74e17b31..584f757a3be 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -12,6 +12,7 @@ import java.util.UUID; import java.util.concurrent.Future; +import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import org.joda.time.DateTime; import org.junit.Before; @@ -101,6 +102,8 @@ public class AnalysisExecutionScheduledTaskImplTest { private IridaFileStorageUtility iridaFileStorageUtility; + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + /** * Sets up variables for tests. */ @@ -110,7 +113,7 @@ public void setup() { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); analysisSubmission = AnalysisSubmission.builder(workflowId) .name("my analysis") @@ -671,7 +674,7 @@ public void testCleanupAnalysisSubmissionsCompletedCleanedCleaningSuccess() thro public void testCleanupAnalysisSubmissionsCompletedOverOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -698,7 +701,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedCleanupZeroSuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZERO), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -725,7 +728,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZER public void testCleanupAnalysisSubmissionsCompletedUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -752,7 +755,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedOverUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); From 3562cf7ec11af7f7ca7dc3a4400a310d9dc6e6e9 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 17 Dec 2020 11:12:56 -0600 Subject: [PATCH 186/655] Added overloaded method getTemporaryFile which accepts a path and prefix. Added params to javadoc --- .../AnalysisSubmissionTempFile.java | 1 - .../IridaFileStorageAwsUtilityImpl.java | 32 +++++++++++++++++++ .../IridaFileStorageAzureUtilityImpl.java | 29 ++++++++++++++++- .../IridaFileStorageLocalUtilityImpl.java | 12 +++++++ .../filesystem/IridaFileStorageUtility.java | 10 ++++++ .../AnalysisCollectionServiceGalaxy.java | 12 ++++--- 6 files changed, 90 insertions(+), 6 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java index c111739cd1f..36a330a86a2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java @@ -7,7 +7,6 @@ import javax.validation.constraints.NotNull; import ca.corefacility.bioinformatics.irida.model.IridaThing; -import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; /** * A temporary file which required by an {@link AnalysisSubmission} when diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 30785be5e98..c4474d6b2c4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -86,6 +86,38 @@ public IridaTemporaryFile getTemporaryFile(Path file) { } } + /** + * {@inheritDoc} + */ + @Override + public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { + String perm = "rwxrwxr-x"; + try { + logger.trace("Getting file from aws s3 [" + file.toString() + "]"); + Path tempDirectory = Files.createTempDirectory(prefix + "-aws-tmp-"); + Path tempFile = tempDirectory.resolve(file.getFileName() + .toString()); + Set permissions = PosixFilePermissions.fromString(perm); + Files.setPosixFilePermissions(tempDirectory, permissions); + + try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); + S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { + org.apache.commons.io.FileUtils.copyInputStreamToFile(s3ObjectInputStream, tempFile.toFile()); + } catch (AmazonServiceException e) { + logger.error(e.getMessage()); + throw new StorageException("Unable to read object from aws s3 bucket", e); + } + + return new IridaTemporaryFile(tempFile, tempDirectory); + } catch (FileNotFoundException e) { + logger.error(e.getMessage()); + throw new StorageException("Unable to resolve temp file in temp directory", e); + } catch (IOException e) { + logger.error(e.getMessage()); + throw new StorageException("Unable to create temp directory", e); + } + } + /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 388a5e01c57..899ab6c0754 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -57,7 +57,6 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St * {@inheritDoc} */ @Override - public IridaTemporaryFile getTemporaryFile(Path file) { String perm = "rwxrwxr-x"; try { @@ -82,6 +81,34 @@ public IridaTemporaryFile getTemporaryFile(Path file) { } } + /** + * {@inheritDoc} + */ + @Override + public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { + String perm = "rwxrwxr-x"; + try { + // We set the blobClient "path" to which file we want to get + BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + try (InputStream initialStream = blobClient.openInputStream()) { + logger.trace("Getting file from azure [" + file.toString() + "]"); + Path tempDirectory = Files.createTempDirectory(prefix + "-azure-tmp-"); + Path tempFile = tempDirectory.resolve(file.getFileName() + .toString()); + org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); + Set permissions = PosixFilePermissions.fromString(perm); + Files.setPosixFilePermissions(tempDirectory, permissions); + return new IridaTemporaryFile(tempFile, tempDirectory); + } catch (IOException e) { + logger.error(e.getMessage()); + throw new StorageException(e.getMessage()); + } + } catch (BlobStorageException e) { + logger.error("Couldn't find file on azure [" + e + "]"); + throw new StorageException("Unable to locate file on azure", e); + } + } + /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 8a65976a1b9..b8d2c4f93b2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -44,6 +44,18 @@ public IridaTemporaryFile getTemporaryFile(Path file) { return new IridaTemporaryFile(file, null); } + /** + * {@inheritDoc} + */ + @Override + public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { + /* + For the local storage we don't need a temp directory with + a prefix so we just call the method above + */ + return getTemporaryFile(file); + } + /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index c2274036337..904f2e70c03 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -26,6 +26,16 @@ public interface IridaFileStorageUtility { */ public IridaTemporaryFile getTemporaryFile(Path file); + /** + * Overloaded method to get a file from storage and + * add prefix to directory + * + * @param file The {@link Path} to the file + * @param prefix The {@link String} prefix to add to the directory name + * @return {@link IridaTemporaryFile} which includes the file and optional temporary directory + */ + public IridaTemporaryFile getTemporaryFile(Path file, String prefix); + /** * Delete temporary downloaded file and/or directory. * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index 29ffff713c2..75b59a13e8e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -49,11 +49,15 @@ public class AnalysisCollectionServiceGalaxy { /** * Builds a new {@link AnalysisCollectionServiceGalaxy} with the given * information. - * - * @param galaxyHistoriesService A GalaxyHistoriesService for interacting with - * Galaxy Histories. + * + * @param galaxyHistoriesService A GalaxyHistoriesService for interacting with + * Galaxy Histories. + * @param iridaFileStorageUtility The file storage utility implementation + * @param analysisSubmissionTempFileRepository {@link AnalysisSubmissionTempFileRepository} for {@link AnalysisSubmissionTempFile} objects */ - public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, IridaFileStorageUtility iridaFileStorageUtility, AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { + public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, + IridaFileStorageUtility iridaFileStorageUtility, + AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { this.galaxyHistoriesService = galaxyHistoriesService; this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; From e80234be450d37fb1c14d8d6b359b3cc9ccdcdd5 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 17 Dec 2020 12:55:21 -0600 Subject: [PATCH 187/655] Updated getTemporary call with passing an 'analysis' prefix so we can delineate between regular object store temp files and analysis ones --- .../workspace/galaxy/AnalysisCollectionServiceGalaxy.java | 6 +++--- .../service/impl/AnalysisExecutionScheduledTaskImpl.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index 75b59a13e8e..dff0a6a9cf3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -88,7 +88,7 @@ public CollectionResponse uploadSequenceFilesSingleEnd( Map samplesMap = new HashMap<>(); for (Sample sample : sampleSequenceFiles.keySet()) { SingleEndSequenceFile sequenceFile = sampleSequenceFiles.get(sample); - IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile()); + IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile(), "analysis"); samplesMap.put(iridaTemporaryFile.getFile(), sample); /* @@ -156,8 +156,8 @@ public CollectionResponse uploadSequenceFilesPaired( SequenceFile fileForward = sequenceFilePair.getForwardSequenceFile(); SequenceFile fileReverse = sequenceFilePair.getReverseSequenceFile(); - IridaTemporaryFile iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile()); - IridaTemporaryFile iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile()); + IridaTemporaryFile iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile(), "analysis"); + IridaTemporaryFile iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile(), "analysis"); samplesMapPairForward.put(sample, iridaTemporaryFileForward.getFile()); samplesMapPairReverse.put(sample, iridaTemporaryFileReverse.getFile()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index d96783bae54..f23282723f7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -321,9 +321,9 @@ private Future handleWorkflowStatus(GalaxyWorkflowStatus wor Cleanup any files that were downloaded from an object store to run an analysis and remove the analysis submission temp file record from the database. */ - if (!iridaFileStorageUtility.storageTypeIsLocal()) { - List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( - analysisSubmission.getId()); + List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( + analysisSubmission.getId()); + if(analysisSubmissionTempFiles.size() > 0) { logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + " temporary files downloaded from object store."); for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( From 3b937b1d4d2dc9542d7dfbc5fd56707341025a2f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 17 Dec 2020 14:01:04 -0600 Subject: [PATCH 188/655] Changed to check if storage type isn't local before attempting to cleanup temp files --- .../irida/service/impl/AnalysisExecutionScheduledTaskImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index f23282723f7..55c86a590dc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -321,9 +321,9 @@ private Future handleWorkflowStatus(GalaxyWorkflowStatus wor Cleanup any files that were downloaded from an object store to run an analysis and remove the analysis submission temp file record from the database. */ + if(!iridaFileStorageUtility.storageTypeIsLocal()) { List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( analysisSubmission.getId()); - if(analysisSubmissionTempFiles.size() > 0) { logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + " temporary files downloaded from object store."); for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( From b47ef804ace2d29d02bac15593fc7f7e5129e205 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 17 Dec 2020 15:08:26 -0600 Subject: [PATCH 189/655] Fixed formatting. Updated azure and aws file storage utilities to check for read/write access. Removed checkconnectivity method and refactored into checkwriteaccess. Injected storageType bean into iridaFileStorageUtility bean. Rempved system.exit(1) calls and replaced with uncaught storageexceptions which will allow the server software to deal with it as it sees fit --- .../services/IridaApiServicesConfig.java | 17 ++---- .../IridaFileStorageAwsUtilityImpl.java | 39 ++++++++------ .../IridaFileStorageAzureUtilityImpl.java | 45 +++++++++------- .../IridaFileStorageLocalUtilityImpl.java | 54 +++++++++---------- .../filesystem/IridaFileStorageUtility.java | 11 ---- 5 files changed, 79 insertions(+), 87 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index d182d6695f8..8d29c77f975 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -321,31 +321,22 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { * @return A new {@link IridaFileStorageUtility} implementation. */ @Bean(name = "iridaFileStorageUtility") - public IridaFileStorageUtility iridaFileStorageService() { + public IridaFileStorageUtility iridaFileStorageService(StorageType storageType) { IridaFileStorageUtility iridaFileStorageUtility; - StorageType st = StorageType.fromString(storageType); - if (st.equals(StorageType.AWS)) { + if (storageType.equals(StorageType.AWS)) { iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(awsBucketName, awsBucketRegion, awsAccessKey, awsSecretKey); - } else if (st.equals(StorageType.AZURE)) { + } else if (storageType.equals(StorageType.AZURE)) { iridaFileStorageUtility = new IridaFileStorageAzureUtilityImpl(containerUrl, sasToken, containerName); } else { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); } - // Check if connection is valid to local file system or cloud provider - try { - iridaFileStorageUtility.checkConnectivity(); - } catch (StorageException e) { - // Log the error and exit so startup does not continue - logger.error("Unable to start up IRIDA! ", e); - System.exit(1); - } IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); return iridaFileStorageUtility; } - @Bean(name= "storageType") + @Bean(name = "storageType") public StorageType storageType() { return StorageType.fromString(storageType); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index abd461b3b73..ac3421c0a73 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -7,6 +7,7 @@ import java.nio.file.StandardOpenOption; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; @@ -309,24 +310,32 @@ public byte[] readAllBytes(Path file) { return bytes; } + /** + * {@inheritDoc} + */ @Override - public boolean checkConnectivity() throws StorageException { - try { - if (s3.doesBucketExistV2(bucketName)) { - logger.debug("Successfully connected to aws s3 bucket " + bucketName); - return true; - } else { - throw new StorageException( - "Unable to connect to aws s3 bucket. Please check that your credentials are valid and that the bucket " + bucketName + " exists."); + public boolean checkWriteAccess(Path baseDirectory) { + // get list of bucket permission + List bucketPermissions = s3.getBucketAcl(bucketName) + .getGrantsAsList() + .stream() + .distinct() + .map(t -> t.getPermission() + .toString()) + .collect(Collectors.toList()); + + if (bucketPermissions.size() > 0) { + // check read/write or full control permission + if (!((bucketPermissions.contains("READ") && bucketPermissions.contains("WRITE")) + || (bucketPermissions.contains("FULL_CONTROL")))) { + throw new StorageException("Unable to read and/or write to bucket " + bucketName + + ". Please check bucket has both read and write permissions."); } - } catch (AmazonServiceException e) { + return true; + } else { throw new StorageException( - "Unable to connect to aws s3 bucket. Please check that your credentials are valid and that the bucket " + bucketName + " exists."); + "Unable to locate bucket " + bucketName + ". Please check your credentials and that the bucket " + + bucketName + " exists."); } } - - @Override - public boolean checkWriteAccess(Path baseDirectory) { - return true; - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 7456c2e8d5c..674ab0ac21f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -306,28 +307,34 @@ private String getAzureFileAbsolutePath(Path file) { return absolutePath; } + /** + * {@inheritDoc} + */ @Override - public boolean checkConnectivity() throws StorageException { + public boolean checkWriteAccess(Path baseDirectory) { String containerName = containerClient.getBlobContainerName(); try { - // Make an api request to check if we can list the blobs in the container - containerClient.listBlobs().forEach(blob -> - blob.getName()); - logger.debug("Successfully connected to azure container " + containerName); - return true; - - } catch (BlobStorageException e) { - /* - Throw an exception which is caught at startup advising the user that the - connection was not successful to the azure container - */ - throw new StorageException( - "Unable to connect to azure container. Please check that your credentials are valid and that the container " + containerName + " exists."); + Path tempDirectory = Files.createTempDirectory(null); + Path tempFile = tempDirectory.resolve("testAzureContainerReadWrite.txt"); + // write a line + Files.write(tempFile, "Azure check read/write permissions.\n".getBytes(StandardCharsets.UTF_8)); + try { + // Upload and delete file to check if container has read/write access + BlobClient blobClient = containerClient.getBlobClient( + getAzureFileAbsolutePath(baseDirectory) + "/" + tempFile.getFileName()); + blobClient.uploadFromFile(tempFile.toString(), false); + blobClient.delete(); + return true; + } catch (BlobStorageException e) { + throw new StorageException("Unable to read and/or write to container " + containerName + + ". Please check container has both read and write permissions.", e); + } finally { + // Cleanup the temporary file on the server + Files.delete(tempFile); + org.apache.commons.io.FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } catch (IOException e) { + throw new StorageException("Unable to clean up temporary file", e); } } - - @Override - public boolean checkWriteAccess(Path baseDirectory) { - return true; - } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index d7893d5384a..2071f2fb565 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -26,11 +26,11 @@ * Component implementation of file utitlities for local storage */ @Component -public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility{ +public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); @Autowired - public IridaFileStorageLocalUtilityImpl(){ + public IridaFileStorageLocalUtilityImpl() { } /** @@ -46,11 +46,14 @@ public IridaTemporaryFile getTemporaryFile(Path file) { */ @Override public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTemporaryFile) { - if(iridaTemporaryFile.getFile() != null) { - logger.trace("File resides on local filesystem. Not cleaning up file [" + iridaTemporaryFile.getFile().toString() + "]"); + if (iridaTemporaryFile.getFile() != null) { + logger.trace("File resides on local filesystem. Not cleaning up file [" + iridaTemporaryFile.getFile() + .toString() + "]"); } - if(iridaTemporaryFile.getDirectoryPath() != null) { - logger.trace("Directory resides on local filesystem. Not cleaning up directory [" + iridaTemporaryFile.getDirectoryPath().toString() + "]"); + if (iridaTemporaryFile.getDirectoryPath() != null) { + logger.trace("Directory resides on local filesystem. Not cleaning up directory [" + + iridaTemporaryFile.getDirectoryPath() + .toString() + "]"); } } @@ -61,7 +64,7 @@ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTempora public String getFileSize(Path file) { String fileSize = "N/A"; try { - if(file != null) { + if (file != null) { fileSize = FileUtils.humanReadableByteCount(Files.size(file), true); } } catch (NoSuchFileException e) { @@ -105,7 +108,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque /** * {@inheritDoc} */ - public boolean storageTypeIsLocal(){ + public boolean storageTypeIsLocal() { return true; } @@ -114,7 +117,8 @@ public boolean storageTypeIsLocal(){ */ public String getFileName(Path file) { String fileName = ""; - fileName = file.getFileName().toString(); + fileName = file.getFileName() + .toString(); return fileName; } @@ -133,7 +137,7 @@ public boolean fileExists(Path file) { public InputStream getFileInputStream(Path file) { try { return Files.newInputStream(file, StandardOpenOption.READ); - } catch(IOException e) { + } catch (IOException e) { throw new FileProcessorException("could not read file", e); } } @@ -146,8 +150,8 @@ public boolean isGzipped(Path file) throws IOException { try (InputStream is = getFileInputStream(file)) { byte[] bytes = new byte[2]; is.read(bytes); - return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) - && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); + return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC + >> 8))); } } @@ -197,8 +201,7 @@ public String getFileExtension(List sequencingObject selectedExtension = currentExtensionOpt.get(); } else if (selectedExtension != currentExtensionOpt.get()) { throw new IOException( - "Extensions of files do not match " + currentExtension + " vs " - + selectedExtension); + "Extensions of files do not match " + currentExtension + " vs " + selectedExtension); } } } @@ -214,24 +217,19 @@ public byte[] readAllBytes(Path file) { byte[] bytes = new byte[0]; try { bytes = Files.readAllBytes(file); - } catch (IOException e) - { + } catch (IOException e) { logger.error("Unable to read file"); } return bytes; } - @Override - public boolean checkConnectivity() { - // Since it's the local filesystem we just return true - return true; - } - + /** + * {@inheritDoc} + */ @Override public boolean checkWriteAccess(Path baseDirectory) { if (!Files.exists(baseDirectory)) { - logger.error("Cannot continue startup; base directory " + baseDirectory + " does not exist!"); - System.exit(1); + throw new StorageException("Cannot continue startup; base directory " + baseDirectory + " does not exist!"); } else { try { // Check if basedirectory path is writeable by creating a temp file and then removing it @@ -243,18 +241,16 @@ public boolean checkWriteAccess(Path baseDirectory) { // Cleanup the temp file created in the directory Files.delete(tempFile); } catch (IOException e) { - logger.error("An I/O error occurred while attempting to remove temp file ", e); + throw new StorageException("An I/O error occurred while attempting to remove temp file ", e); } if (!directoryWriteable) { // Log the error and exit so startup does not continue - logger.error("Cannot continue startup; base directory " + baseDirectory + throw new StorageException("Cannot continue startup; base directory " + baseDirectory + " does not have write access! Please check directory permissions."); - System.exit(1); } } catch (IOException e) { - logger.error("Unable to create temporary file. Please check directory permissions", e); - System.exit(1); + throw new StorageException("Unable to create temporary file. Please check directory permissions", e); } } /* diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 5f0b37ea358..28b1c304930 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -125,17 +125,6 @@ public interface IridaFileStorageUtility { */ public byte[] readAllBytes(Path file); - /** - * Check the connectivity to the file storage - * mechanism. For cloud services we must make an - * api call to check if containers/buckets exist. - * - * @return Connection valid or not - * @throws StorageException if connection failed - */ - public boolean checkConnectivity() throws StorageException; - - /** * Check if the given directory is writable * From 90dd0c0e88e1c79d052903e76e3a2377c5317a58 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 17 Dec 2020 15:42:25 -0600 Subject: [PATCH 190/655] Removed unused imports --- .../irida/config/services/IridaApiServicesConfig.java | 1 - .../irida/repositories/filesystem/IridaFileStorageUtility.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 8d29c77f975..3590af4543b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -8,7 +8,6 @@ import ca.corefacility.bioinformatics.irida.config.services.conditions.NreplServerSpringCondition; import ca.corefacility.bioinformatics.irida.config.services.scheduled.IridaScheduledTasksConfig; import ca.corefacility.bioinformatics.irida.config.workflow.IridaWorkflowsConfig; -import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 28b1c304930..2b990a1a775 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -5,7 +5,6 @@ import java.nio.file.Path; import java.util.List; -import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; From 6ee8908d303ce05b9aa2ee95811bbc06c4c8cc46 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 2 Feb 2021 16:35:20 -0600 Subject: [PATCH 191/655] Updated sistr and bio hansel sample updaters to use an input stream to get file required to support use of object store --- .../updater/impl/BioHanselSampleUpdater.java | 14 ++-- .../updater/impl/SISTRSampleUpdater.java | 78 ++++++++++--------- .../impl/unit/BioHanselSampleUpdaterTest.java | 7 +- .../impl/unit/SISTRSampleUpdaterTest.java | 6 +- 4 files changed, 62 insertions(+), 43 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java index f224638c79e..a3dac2c5f9e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java @@ -11,6 +11,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.AnalysisSampleUpdater; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.fasterxml.jackson.core.type.TypeReference; @@ -24,6 +25,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.*; @@ -47,11 +49,13 @@ public class BioHanselSampleUpdater implements AnalysisSampleUpdater { // @formatter:on private MetadataTemplateService metadataTemplateService; private SampleService sampleService; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public BioHanselSampleUpdater(MetadataTemplateService metadataTemplateService, SampleService sampleService) { + public BioHanselSampleUpdater(MetadataTemplateService metadataTemplateService, SampleService sampleService, IridaFileStorageUtility iridaFileStorageUtility) { this.metadataTemplateService = metadataTemplateService; this.sampleService = sampleService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -78,9 +82,9 @@ public void update(Collection samples, AnalysisSubmission analysis) thro Path filePath = aof.getFile(); Map stringEntries = new HashMap<>(); - try { - @SuppressWarnings("resource") String jsonText = new Scanner( - new BufferedReader(new FileReader(filePath.toFile()))).useDelimiter("\\Z") + + try(InputStream inputStream = iridaFileStorageUtility.getFileInputStream(filePath)) { + @SuppressWarnings("resource") String jsonText = new Scanner(inputStream).useDelimiter("\\Z") .next(); ObjectMapper mapper = new ObjectMapper(); List> maps = mapper.readValue(jsonText, new TypeReference>>() { @@ -113,10 +117,10 @@ public void update(Collection samples, AnalysisSubmission analysis) thro } else { throw new PostProcessingException(filePath + " not correctly formatted. Expected valid JSON."); } - } catch (IOException e) { throw new PostProcessingException("Error parsing JSON from " + filePath, e); } + } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java index 3674f60ec1f..602e9671ad6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java @@ -2,6 +2,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.exceptions.PostProcessingException; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; import ca.corefacility.bioinformatics.irida.model.sample.metadata.PipelineProvidedMetadataEntry; @@ -11,6 +12,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.AnalysisSampleUpdater; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; @@ -23,6 +25,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.*; @@ -36,6 +39,7 @@ public class SISTRSampleUpdater implements AnalysisSampleUpdater { private MetadataTemplateService metadataTemplateService; private IridaWorkflowsService iridaWorkflowsService; private SampleService sampleService; + private IridaFileStorageUtility iridaFileStorageUtility; // @formatter:off private static Map SISTR_FIELDS = ImmutableMap.builder() @@ -53,10 +57,11 @@ public class SISTRSampleUpdater implements AnalysisSampleUpdater { @Autowired public SISTRSampleUpdater(MetadataTemplateService metadataTemplateService, SampleService sampleService, - IridaWorkflowsService iridaWorkflowsService) { + IridaWorkflowsService iridaWorkflowsService, IridaFileStorageUtility iridaFileStorageUtility) { this.metadataTemplateService = metadataTemplateService; this.sampleService = sampleService; this.iridaWorkflowsService = iridaWorkflowsService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -77,45 +82,46 @@ public void update(Collection samples, AnalysisSubmission analysis) thro IridaWorkflow iridaWorkflow = iridaWorkflowsService.getIridaWorkflow(analysis.getWorkflowId()); String workflowVersion = iridaWorkflow.getWorkflowDescription().getVersion(); - //Read the JSON file from SISTR output - @SuppressWarnings("resource") - String jsonFile = new Scanner(new BufferedReader(new FileReader(filePath.toFile()))).useDelimiter("\\Z") - .next(); + try(InputStream inputStream = iridaFileStorageUtility.getFileInputStream(filePath)) { + + //Read the JSON file from SISTR output + @SuppressWarnings("resource") + String jsonFile = new Scanner(inputStream).useDelimiter("\\Z") + .next(); + // map the results into a Map + ObjectMapper mapper = new ObjectMapper(); + List> sistrResults = mapper + .readValue(jsonFile, new TypeReference>>() { + }); + + if (sistrResults.size() > 0) { + Map result = sistrResults.get(0); + + //loop through each of the requested fields and save the entries + SISTR_FIELDS.entrySet().forEach(e -> { + if (result.containsKey(e.getKey())) { + Object valueObject = result.get(e.getKey()); + String value = (valueObject != null ? valueObject.toString() : ""); + PipelineProvidedMetadataEntry metadataEntry = new PipelineProvidedMetadataEntry(value, "text", + analysis); + stringEntries.put(e.getValue() + " (v" + workflowVersion + ")", metadataEntry); + } + }); + + // convert string map into metadata fields + Set metadataSet = metadataTemplateService.convertMetadataStringsToSet(stringEntries); - // map the results into a Map - ObjectMapper mapper = new ObjectMapper(); - List> sistrResults = mapper - .readValue(jsonFile, new TypeReference>>() { + //save metadata back to sample + samples.forEach(s -> { + sampleService.mergeSampleMetadata(s, metadataSet); }); - if (sistrResults.size() > 0) { - Map result = sistrResults.get(0); - - //loop through each of the requested fields and save the entries - SISTR_FIELDS.entrySet().forEach(e -> { - if (result.containsKey(e.getKey())) { - Object valueObject = result.get(e.getKey()); - String value = (valueObject != null ? valueObject.toString() : ""); - PipelineProvidedMetadataEntry metadataEntry = new PipelineProvidedMetadataEntry(value, "text", - analysis); - stringEntries.put(e.getValue() + " (v" + workflowVersion + ")", metadataEntry); - } - }); - - // convert string map into metadata fields - Set metadataSet = metadataTemplateService.convertMetadataStringsToSet(stringEntries); - - //save metadata back to sample - samples.forEach(s -> { - sampleService.mergeSampleMetadata(s, metadataSet); - }); - - } else { - throw new PostProcessingException("SISTR results for file are not correctly formatted"); + } else { + throw new PostProcessingException("SISTR results for file are not correctly formatted"); + } + } catch (IOException e) { + throw new PostProcessingException("Error parsing JSON from SISTR results", e); } - - } catch (IOException e) { - throw new PostProcessingException("Error parsing JSON from SISTR results", e); } catch (IridaWorkflowNotFoundException e) { throw new PostProcessingException("Workflow is not found", e); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java index ceaf6597aa1..57d129be4e2 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java @@ -2,6 +2,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.AnalysisAlreadySetException; import ca.corefacility.bioinformatics.irida.exceptions.PostProcessingException; +import ca.corefacility.bioinformatics.irida.model.IridaClientDetails; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField; import ca.corefacility.bioinformatics.irida.model.sample.Sample; @@ -11,6 +12,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.BioHanselSampleUpdater; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.google.common.collect.ImmutableMap; @@ -37,12 +40,14 @@ public class BioHanselSampleUpdaterTest { private BioHanselSampleUpdater bioHanselSampleUpdater; private MetadataTemplateService metadataTemplateService; private SampleService sampleService; + private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { metadataTemplateService = mock(MetadataTemplateService.class); sampleService = mock(SampleService.class); - bioHanselSampleUpdater = new BioHanselSampleUpdater(metadataTemplateService, sampleService); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + bioHanselSampleUpdater = new BioHanselSampleUpdater(metadataTemplateService, sampleService, iridaFileStorageUtility); } @SuppressWarnings("unchecked") diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java index 3515f3d97f8..f8f62a44327 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java @@ -13,6 +13,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.description.IridaWorkflowDescription; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.SISTRSampleUpdater; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; @@ -44,6 +46,7 @@ public class SISTRSampleUpdaterTest { private IridaWorkflowsService iridaWorkflowsService; private IridaWorkflow iridaWorkflow; private IridaWorkflowDescription iridaWorkflowDescription; + private IridaFileStorageUtility iridaFileStorageUtility; private UUID uuid = UUID.randomUUID(); @@ -54,8 +57,9 @@ public void setUp() throws IridaWorkflowNotFoundException { iridaWorkflowsService = mock(IridaWorkflowsService.class); iridaWorkflow = mock(IridaWorkflow.class); iridaWorkflowDescription = mock(IridaWorkflowDescription.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - updater = new SISTRSampleUpdater(metadataTemplateService, sampleService, iridaWorkflowsService); + updater = new SISTRSampleUpdater(metadataTemplateService, sampleService, iridaWorkflowsService, iridaFileStorageUtility); when(iridaWorkflowsService.getIridaWorkflow(uuid)).thenReturn(iridaWorkflow); From 8f41ddcdc2900c9d36c460d4acb493a59b689cd3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 3 Feb 2021 12:47:45 -0600 Subject: [PATCH 192/655] Updated Sistr sample updater test expected condition --- .../results/updater/impl/unit/SISTRSampleUpdaterTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java index f8f62a44327..16d142577fb 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java @@ -13,6 +13,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.description.IridaWorkflowDescription; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.SISTRSampleUpdater; +import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; @@ -215,7 +216,7 @@ public void testUpdaterBadFile() throws PostProcessingException, AnalysisAlready updater.update(Lists.newArrayList(sample), submission); } - @Test(expected = PostProcessingException.class) + @Test(expected = FileProcessorException.class) public void testUpdaterNoFile() throws PostProcessingException, AnalysisAlreadySetException { Path outputPath = Paths.get("src/test/resources/files/not_really_a_file.txt"); From d05c73d1a4f75328dc2552c7a0df3eabdf775fae Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 3 Feb 2021 14:15:52 -0600 Subject: [PATCH 193/655] Removed unused imports --- .../pipeline/results/updater/impl/BioHanselSampleUpdater.java | 2 -- .../pipeline/results/updater/impl/SISTRSampleUpdater.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java index a3dac2c5f9e..b5c2b51361e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java @@ -22,8 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.BufferedReader; -import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java index 602e9671ad6..9139f8e6d69 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java @@ -2,7 +2,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.exceptions.PostProcessingException; -import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; import ca.corefacility.bioinformatics.irida.model.sample.metadata.PipelineProvidedMetadataEntry; @@ -22,8 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.BufferedReader; -import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; From 40d9ef0f43abdf0162b1bba7c3f38ac424ef4876 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 12 Feb 2021 14:25:33 -0600 Subject: [PATCH 194/655] Refactored cleanup of temporary files used by a submission and the associated analysissubmissiontempfile objects into a private method. Updated storage utilities to check which storagetype is set to determine if storage type is local or not. Updated setting of pointer after reading first line of a file to set to 0 if there is no text in the file --- .../IridaFileStorageAwsUtilityImpl.java | 6 ++- .../IridaFileStorageAzureUtilityImpl.java | 6 ++- .../IridaFileStorageLocalUtilityImpl.java | 6 ++- .../web/analysis/AnalysisAjaxController.java | 8 +++- .../AnalysisExecutionScheduledTaskImpl.java | 47 +++++++++++++------ 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 9079754c601..c9d77bc4868 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -19,6 +19,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -45,6 +46,9 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private BasicAWSCredentials awsCreds; private AmazonS3 s3; + @Autowired + private StorageType storageType; + @Autowired public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); @@ -196,7 +200,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque */ @Override public boolean storageTypeIsLocal() { - return false; + return storageType.equals(StorageType.LOCAL); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 14636484cc3..ddf5e117704 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -46,6 +47,9 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; + @Autowired + private StorageType storageType; + @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl) @@ -189,7 +193,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque */ @Override public boolean storageTypeIsLocal() { - return false; + return storageType.equals(StorageType.LOCAL); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 159ef7d787a..cc3d20e94ad 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -18,6 +18,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -32,6 +33,9 @@ public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); + @Autowired + private StorageType storageType; + @Autowired public IridaFileStorageLocalUtilityImpl() { } @@ -124,7 +128,7 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque * {@inheritDoc} */ public boolean storageTypeIsLocal() { - return true; + return storageType.equals(StorageType.LOCAL); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index bffc98fd7fd..0c6c010eefc 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -469,8 +469,12 @@ private void addFirstLine(AnalysisOutputFileInfo info, AnalysisOutputFile aof) { try(BufferedReader reader = new BufferedReader(new InputStreamReader(aof.getFileInputStream(), "UTF-8"))) { String firstLineText = reader.readLine(); info.setFirstLine(firstLineText); - // Set the pointer to the beginning of the next line. - info.setFilePointer(Long.valueOf(firstLineText.getBytes().length) + 1); + if(firstLineText != null ){ + // Set the pointer to the beginning of the next line. + info.setFilePointer(Long.valueOf(firstLineText.getBytes().length) + 1); + } else { + info.setFilePointer(0L); + } } catch (IOException e) { logger.error("Could not get file input stream '" + aofFile + "' " + e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index ea5fe2f3ca2..7c9edfc9e5d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -16,6 +16,7 @@ import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; +import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -26,6 +27,7 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; +import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; @@ -191,6 +193,10 @@ public Set> monitorRunningAnalyses() { logger.error("Error checking state for " + analysisSubmission, e); analysisSubmission.setAnalysisState(AnalysisState.ERROR); submissions.add(new AsyncResult<>(analysisSubmissionRepository.save(analysisSubmission))); + + // Clean up any downloaded temp files + cleanupTemporaryDownloadedFiles(analysisSubmission); + if (analysisSubmission.getEmailPipelineResultError()) { emailController.sendPipelineStatusEmail(analysisSubmission); } @@ -317,21 +323,8 @@ private Future handleWorkflowStatus(GalaxyWorkflowStatus wor */ if (finalWorkflowStatusSet) { - /* - Cleanup any files that were downloaded from an object store to run an analysis and - remove the analysis submission temp file record from the database. - */ - if(!iridaFileStorageUtility.storageTypeIsLocal()) { - List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( - analysisSubmission.getId()); - logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + " temporary files downloaded from object store."); - for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( - new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), - analysisSubmissionTempFile.getFileDirectoryPath())); - analysisSubmissionTempFileRepository.delete(analysisSubmissionTempFile); - } - } + // Clean up any downloaded temp files + cleanupTemporaryDownloadedFiles(analysisSubmission); /* If the analysis has finished with an error or completed successfully @@ -380,4 +373,28 @@ public Set> cleanupAnalysisSubmissions() { return cleanedSubmissions; } } + + /** + * Cleanup any temporary downloaded files and cleanup associated {@link AnalysisSubmissionTempFile} objects + * + * @param submission The analysis submission to clean up temporary downloaded files for + */ + private void cleanupTemporaryDownloadedFiles(AnalysisSubmission submission) { + /* + Cleanup any files that were downloaded from an object store to run an analysis and + remove the analysis submission temp file record from the database. + */ + if (!iridaFileStorageUtility.storageTypeIsLocal()) { + List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( + submission.getId()); + logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + + " temporary files downloaded from object store."); + for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( + new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), + analysisSubmissionTempFile.getFileDirectoryPath())); + analysisSubmissionTempFileRepository.delete(analysisSubmissionTempFile); + } + } + } } From 1e584140a8b34277397357648c16511bb4bac582 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 12 Feb 2021 15:17:30 -0600 Subject: [PATCH 195/655] Updated to use storagetype when checking for storage type --- .../AnalysisExecutionServiceConfig.java | 7 ++++++- .../AnalysisScheduledTaskConfig.java | 6 +++++- .../IridaFileStorageAwsUtilityImpl.java | 12 ----------- .../IridaFileStorageAzureUtilityImpl.java | 12 ----------- .../IridaFileStorageLocalUtilityImpl.java | 10 ---------- .../filesystem/IridaFileStorageUtility.java | 7 ------- .../AnalysisCollectionServiceGalaxy.java | 9 ++++++--- .../AnalysisExecutionScheduledTaskImpl.java | 9 +++++---- .../AnalysisExecutionServiceTestConfig.java | 7 ++++++- .../integration/SNVPhylAnalysisIT.java | 6 +++++- .../integration/GalaxyJobErrorsServiceIT.java | 7 ++++++- .../AnalysisExecutionScheduledTaskImplIT.java | 12 ++++++++--- ...nalysisExecutionScheduledTaskImplTest.java | 20 ++++++++++++++----- 13 files changed, 63 insertions(+), 61 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java index edad2b0f94d..6752b1a5fb5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java @@ -17,6 +17,7 @@ import com.google.common.collect.Lists; import ca.corefacility.bioinformatics.irida.config.services.IridaPluginConfig; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.results.impl.AnalysisSubmissionSampleProcessorImpl; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.AnalysisSampleUpdater; @@ -113,6 +114,9 @@ public class AnalysisExecutionServiceConfig { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; + @Autowired + private StorageType storageType; + private List loadPluginAnalysisSampleUpdaters() { List pluginUpdaters = Lists.newLinkedList(); @@ -179,6 +183,7 @@ public AnalysisProvenanceServiceGalaxy analysisProvenanceService() { @Lazy @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { - return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, + analysisSubmissionTempFileRepository, storageType); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java index 3e1abb7ed83..acf5aaccc34 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java @@ -1,5 +1,6 @@ package ca.corefacility.bioinformatics.irida.config.services.scheduled; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; @@ -58,6 +59,9 @@ public class AnalysisScheduledTaskConfig { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + @Autowired + private StorageType storageType; + /** * Defines the time to clean up in number of days a submission must exist before it is cleaned up. */ @@ -133,7 +137,7 @@ public void cleanupAnalysisSubmissions() { public AnalysisExecutionScheduledTask analysisExecutionScheduledTask() { return new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, cleanupAnalysisSubmissionCondition(), galaxyJobErrorsService, jobErrorRepository, emailController, - analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index c9d77bc4868..452322397c4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -19,7 +19,6 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -46,9 +45,6 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private BasicAWSCredentials awsCreds; private AmazonS3 s3; - @Autowired - private StorageType storageType; - @Autowired public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { this.awsCreds = new BasicAWSCredentials(accessKey, secretKey); @@ -195,14 +191,6 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque } } - /** - * {@inheritDoc} - */ - @Override - public boolean storageTypeIsLocal() { - return storageType.equals(StorageType.LOCAL); - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index ddf5e117704..4172fc18f55 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -23,7 +23,6 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -47,9 +46,6 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; - @Autowired - private StorageType storageType; - @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { this.blobServiceClient = new BlobServiceClientBuilder().endpoint(containerUrl) @@ -188,14 +184,6 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque } } - /** - * {@inheritDoc} - */ - @Override - public boolean storageTypeIsLocal() { - return storageType.equals(StorageType.LOCAL); - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index cc3d20e94ad..33d233f537e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -18,7 +18,6 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -33,8 +32,6 @@ public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); - @Autowired - private StorageType storageType; @Autowired public IridaFileStorageLocalUtilityImpl() { @@ -124,13 +121,6 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque } } - /** - * {@inheritDoc} - */ - public boolean storageTypeIsLocal() { - return storageType.equals(StorageType.LOCAL); - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index d731a10282e..6fc1d1d2747 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -61,13 +61,6 @@ public interface IridaFileStorageUtility { */ public void writeFile(Path source, Path target, Path sequenceFileDir, Path sequenceFileDirWithRevision); - /** - * Returns if the storage type is local or not - * - * @return {@link Boolean#TRUE} if local, {@link Boolean#FALSE} if not. - */ - public boolean storageTypeIsLocal(); - /** * Gets the file name from the storage type that the file * is saved to (azure, aws, or local disk) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index dff0a6a9cf3..cf37777caf1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -2,6 +2,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; import ca.corefacility.bioinformatics.irida.exceptions.UploadException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; @@ -45,6 +46,7 @@ public class AnalysisCollectionServiceGalaxy { private GalaxyHistoriesService galaxyHistoriesService; private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + private StorageType storageType; /** * Builds a new {@link AnalysisCollectionServiceGalaxy} with the given @@ -57,10 +59,11 @@ public class AnalysisCollectionServiceGalaxy { */ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, IridaFileStorageUtility iridaFileStorageUtility, - AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { + AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository, StorageType storageType) { this.galaxyHistoriesService = galaxyHistoriesService; this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; + this.storageType = storageType; } /** @@ -96,7 +99,7 @@ public CollectionResponse uploadSequenceFilesSingleEnd( record to save to the database. The cleanup of these files happens once the final workflow status is set for an analysis. */ - if (!iridaFileStorageUtility.storageTypeIsLocal()) { + if (!storageType.equals(StorageType.LOCAL)) { AnalysisSubmissionTempFile analysisSubmissionTempFile = new AnalysisSubmissionTempFile(submissionId, iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); analysisSubmissionTempFileRepository.save(analysisSubmissionTempFile); @@ -169,7 +172,7 @@ public CollectionResponse uploadSequenceFilesPaired( record to save to the database. The cleanup of these files happens once the final workflow status is set for an analysis. */ - if (!iridaFileStorageUtility.storageTypeIsLocal()) { + if (!storageType.equals(StorageType.LOCAL)) { AnalysisSubmissionTempFile analysisSubmissionTempFileForward = new AnalysisSubmissionTempFile( submissionId, iridaTemporaryFileForward.getFile(), iridaTemporaryFileForward.getDirectoryPath()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index 7c9edfc9e5d..1a1356eedde 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -15,8 +15,8 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; -import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -27,7 +27,6 @@ import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; -import ca.corefacility.bioinformatics.irida.ria.web.analysis.dto.AnalysisOutputFileInfo; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; @@ -62,6 +61,7 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisWorkspaceService analysisWorkspaceService; private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + private StorageType storageType; /** * Builds a new AnalysisExecutionScheduledTaskImpl with the given service @@ -83,7 +83,7 @@ public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisS AnalysisExecutionService analysisExecutionServiceGalaxy, CleanupAnalysisSubmissionCondition cleanupCondition, GalaxyJobErrorsService galaxyJobErrorsService, JobErrorRepository jobErrorRepository, EmailController emailController, AnalysisWorkspaceService analysisWorkspaceService, IridaFileStorageUtility iridaFileStorageUtility, - AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { + AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository, StorageType storageType) { this.analysisSubmissionRepository = analysisSubmissionRepository; this.analysisExecutionService = analysisExecutionServiceGalaxy; this.cleanupCondition = cleanupCondition; @@ -93,6 +93,7 @@ public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisS this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisWorkspaceService = analysisWorkspaceService; this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; + this.storageType = storageType; } /** @@ -384,7 +385,7 @@ private void cleanupTemporaryDownloadedFiles(AnalysisSubmission submission) { Cleanup any files that were downloaded from an object store to run an analysis and remove the analysis submission temp file record from the database. */ - if (!iridaFileStorageUtility.storageTypeIsLocal()) { + if (!storageType.equals(StorageType.LOCAL)) { List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( submission.getId()); logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java index ac03cba8a30..a577204e034 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import ca.corefacility.bioinformatics.irida.config.conditions.NonWindowsPlatformCondition; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; @@ -110,6 +111,9 @@ public class AnalysisExecutionServiceTestConfig { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; + + @Autowired + private StorageType storageType; @Bean public AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor() { @@ -151,7 +155,8 @@ galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy() @Lazy @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { - return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, + analysisSubmissionTempFileRepository, storageType); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java index 9d13b66e827..59744fb394f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java @@ -48,6 +48,7 @@ import ca.corefacility.bioinformatics.irida.config.IridaApiGalaxyTestConfig; import ca.corefacility.bioinformatics.irida.config.conditions.WindowsPlatformCondition; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; @@ -137,6 +138,8 @@ public class SNVPhylAnalysisIT { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + @Autowired + private StorageType storageType; private Path sequenceFilePathA1; private Path sequenceFilePathA2; @@ -183,7 +186,8 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.NEVER_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, + analysisSubmissionTempFileRepository, storageType); Path tempDir = Files.createTempDirectory(rootTempDirectory, "snvphylTest"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java index 85f36726add..141166ca47e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java @@ -25,6 +25,7 @@ import ca.corefacility.bioinformatics.irida.config.IridaApiGalaxyTestConfig; import ca.corefacility.bioinformatics.irida.config.conditions.WindowsPlatformCondition; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; @@ -91,6 +92,9 @@ public class GalaxyJobErrorsServiceIT { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + @Autowired + private StorageType storageType; + @Before public void setup() throws URISyntaxException, IOException { Assume.assumeFalse(WindowsPlatformCondition.isWindows()); @@ -109,7 +113,8 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, + analysisSubmissionTempFileRepository, storageType); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java index f32c654b848..622151c5d83 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java @@ -35,6 +35,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -105,6 +106,9 @@ public class AnalysisExecutionScheduledTaskImplIT { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + @Autowired + private StorageType storageType; + private Path sequenceFilePath; private Path sequenceFilePath2; @@ -128,8 +132,9 @@ public void setup() throws URISyntaxException, IOException { Assume.assumeFalse(WindowsPlatformCondition.isWindows()); analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, - analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, + analysisSubmissionTempFileRepository, storageType); Path sequenceFilePathReal = Paths .get(DatabaseSetupGalaxyITService.class.getResource("testData1.fastq").toURI()); @@ -196,7 +201,8 @@ public void testFullAnalysisRunSuccessWithSampleUpdates() throws Exception { public void testFullAnalysisRunSuccessNoCleanupAge() throws Exception { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, + iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); AnalysisSubmission analysisSubmission = analysisExecutionGalaxyITService.setupSubmissionInDatabase(1L, sequenceFilePath, referenceFilePath, validIridaWorkflowId, false); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index d32ff9fe1d7..d7a51d3e50f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -12,6 +12,7 @@ import java.util.UUID; import java.util.concurrent.Future; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import org.joda.time.DateTime; @@ -92,6 +93,7 @@ public class AnalysisExecutionScheduledTaskImplTest { @Mock private AnalysisWorkspaceService analysisWorkspaceService; + private static final String ANALYSIS_ID = "1"; private static final Long INTERNAL_ID = 1L; private AnalysisSubmission analysisSubmission; @@ -104,6 +106,8 @@ public class AnalysisExecutionScheduledTaskImplTest { private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + private StorageType storageType; + /** * Sets up variables for tests. */ @@ -111,9 +115,11 @@ public class AnalysisExecutionScheduledTaskImplTest { public void setup() { MockitoAnnotations.initMocks(this); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + storageType = StorageType.LOCAL; analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, + analysisSubmissionTempFileRepository, storageType); analysisSubmission = AnalysisSubmission.builder(workflowId) .name("my analysis") @@ -678,7 +684,8 @@ public void testCleanupAnalysisSubmissionsCompletedCleanedCleaningSuccess() thro public void testCleanupAnalysisSubmissionsCompletedOverOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, + iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -705,7 +712,8 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedCleanupZeroSuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZERO), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, + iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -732,7 +740,8 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZER public void testCleanupAnalysisSubmissionsCompletedUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, + iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -759,7 +768,8 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedOverUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, + iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); From eb2d3b4fd2ef1fc91ed99b94fb0574c6bc579ea9 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Feb 2021 12:12:44 -0600 Subject: [PATCH 196/655] Updated storage utility classes to contain storagetype as a property. Updated sistr and biohansel sample updaters to use getFileInputStream from IridaFiles so we don't need to autowire in the iridafilestorageutility. Added query annotation. Removed storage type check when cleaning up analysissubmissiontempfile objects. --- .../AnalysisExecutionServiceConfig.java | 5 +--- .../AnalysisScheduledTaskConfig.java | 6 +---- .../updater/impl/BioHanselSampleUpdater.java | 9 +++---- .../updater/impl/SISTRSampleUpdater.java | 9 +++---- .../AnalysisSubmissionTempFileRepository.java | 2 ++ .../IridaFileStorageAwsUtilityImpl.java | 11 ++++++++ .../IridaFileStorageAzureUtilityImpl.java | 11 ++++++++ .../IridaFileStorageLocalUtilityImpl.java | 11 ++++++++ .../filesystem/IridaFileStorageUtility.java | 7 ++++++ .../AnalysisCollectionServiceGalaxy.java | 9 +++---- .../AnalysisExecutionScheduledTaskImpl.java | 25 ++++++++----------- .../AnalysisExecutionServiceTestConfig.java | 5 +--- .../integration/SNVPhylAnalysisIT.java | 5 +--- .../impl/unit/BioHanselSampleUpdaterTest.java | 7 +----- .../impl/unit/SISTRSampleUpdaterTest.java | 4 +-- .../integration/GalaxyJobErrorsServiceIT.java | 5 +--- .../AnalysisExecutionScheduledTaskImplIT.java | 8 ++---- ...nalysisExecutionScheduledTaskImplTest.java | 13 ++++------ 18 files changed, 77 insertions(+), 75 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java index 6752b1a5fb5..c52f1b0b14a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java @@ -17,7 +17,6 @@ import com.google.common.collect.Lists; import ca.corefacility.bioinformatics.irida.config.services.IridaPluginConfig; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.results.impl.AnalysisSubmissionSampleProcessorImpl; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.AnalysisSampleUpdater; @@ -114,8 +113,6 @@ public class AnalysisExecutionServiceConfig { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; - @Autowired - private StorageType storageType; private List loadPluginAnalysisSampleUpdaters() { List pluginUpdaters = Lists.newLinkedList(); @@ -184,6 +181,6 @@ public AnalysisProvenanceServiceGalaxy analysisProvenanceService() { @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository, storageType); + analysisSubmissionTempFileRepository); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java index acf5aaccc34..3e1abb7ed83 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java @@ -1,6 +1,5 @@ package ca.corefacility.bioinformatics.irida.config.services.scheduled; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; @@ -59,9 +58,6 @@ public class AnalysisScheduledTaskConfig { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Autowired - private StorageType storageType; - /** * Defines the time to clean up in number of days a submission must exist before it is cleaned up. */ @@ -137,7 +133,7 @@ public void cleanupAnalysisSubmissions() { public AnalysisExecutionScheduledTask analysisExecutionScheduledTask() { return new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, cleanupAnalysisSubmissionCondition(), galaxyJobErrorsService, jobErrorRepository, emailController, - analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); + analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java index b5c2b51361e..5f97862fe41 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java @@ -11,9 +11,10 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.AnalysisSampleUpdater; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; @@ -47,13 +48,11 @@ public class BioHanselSampleUpdater implements AnalysisSampleUpdater { // @formatter:on private MetadataTemplateService metadataTemplateService; private SampleService sampleService; - private IridaFileStorageUtility iridaFileStorageUtility; @Autowired - public BioHanselSampleUpdater(MetadataTemplateService metadataTemplateService, SampleService sampleService, IridaFileStorageUtility iridaFileStorageUtility) { + public BioHanselSampleUpdater(MetadataTemplateService metadataTemplateService, SampleService sampleService) { this.metadataTemplateService = metadataTemplateService; this.sampleService = sampleService; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -81,7 +80,7 @@ public void update(Collection samples, AnalysisSubmission analysis) thro Map stringEntries = new HashMap<>(); - try(InputStream inputStream = iridaFileStorageUtility.getFileInputStream(filePath)) { + try(InputStream inputStream = IridaFiles.getFileInputStream(filePath)) { @SuppressWarnings("resource") String jsonText = new Scanner(inputStream).useDelimiter("\\Z") .next(); ObjectMapper mapper = new ObjectMapper(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java index 9139f8e6d69..298f90cc269 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java @@ -11,10 +11,11 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.BuiltInAnalysisTypes; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.AnalysisSampleUpdater; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; @@ -36,7 +37,6 @@ public class SISTRSampleUpdater implements AnalysisSampleUpdater { private MetadataTemplateService metadataTemplateService; private IridaWorkflowsService iridaWorkflowsService; private SampleService sampleService; - private IridaFileStorageUtility iridaFileStorageUtility; // @formatter:off private static Map SISTR_FIELDS = ImmutableMap.builder() @@ -54,11 +54,10 @@ public class SISTRSampleUpdater implements AnalysisSampleUpdater { @Autowired public SISTRSampleUpdater(MetadataTemplateService metadataTemplateService, SampleService sampleService, - IridaWorkflowsService iridaWorkflowsService, IridaFileStorageUtility iridaFileStorageUtility) { + IridaWorkflowsService iridaWorkflowsService) { this.metadataTemplateService = metadataTemplateService; this.sampleService = sampleService; this.iridaWorkflowsService = iridaWorkflowsService; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -79,7 +78,7 @@ public void update(Collection samples, AnalysisSubmission analysis) thro IridaWorkflow iridaWorkflow = iridaWorkflowsService.getIridaWorkflow(analysis.getWorkflowId()); String workflowVersion = iridaWorkflow.getWorkflowDescription().getVersion(); - try(InputStream inputStream = iridaFileStorageUtility.getFileInputStream(filePath)) { + try(InputStream inputStream = IridaFiles.getFileInputStream(filePath)) { //Read the JSON file from SISTR output @SuppressWarnings("resource") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java index d30d82fba4a..52e8e7a40d7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java @@ -2,6 +2,7 @@ import java.util.List; +import org.springframework.data.jpa.repository.Query; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; import ca.corefacility.bioinformatics.irida.repositories.IridaJpaRepository; @@ -18,5 +19,6 @@ public interface AnalysisSubmissionTempFileRepository extends IridaJpaRepository * @param analysisSubmissionId The analysis submission id * @return a list of {@link AnalysisSubmissionTempFile} */ + @Query("FROM AnalysisSubmissionTempFile f WHERE f.analysisSubmissionId = ?1") List findAllByAnalysisSubmissionId(Long analysisSubmissionId); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 452322397c4..17aeeac04f9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -19,6 +19,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -44,6 +45,7 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private String bucketName; private BasicAWSCredentials awsCreds; private AmazonS3 s3; + private StorageType storageType; @Autowired public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { @@ -53,6 +55,7 @@ public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, St .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .build(); this.bucketName = bucketName; + this.storageType = StorageType.AWS; } /** @@ -410,4 +413,12 @@ public boolean checkWriteAccess(Path baseDirectory) { + bucketName + " exists."); } } + + /** + * {@inheritDoc} + */ + @Override + public boolean isStorageTypeLocal() { + return storageType.equals(StorageType.LOCAL); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 4172fc18f55..ef5205762f5 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -45,6 +46,7 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; + private StorageType storageType; @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { @@ -52,6 +54,7 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St .sasToken(sasToken) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + this.storageType = StorageType.AZURE; } /** @@ -407,4 +410,12 @@ public boolean checkWriteAccess(Path baseDirectory) { throw new StorageException("Unable to clean up temporary file", e); } } + + /** + * {@inheritDoc} + */ + @Override + public boolean isStorageTypeLocal() { + return storageType.equals(StorageType.LOCAL); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 33d233f537e..c896aa44ef1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -18,6 +18,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -32,9 +33,11 @@ public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); + private StorageType storageType; @Autowired public IridaFileStorageLocalUtilityImpl() { + this.storageType = StorageType.LOCAL; } /** @@ -301,4 +304,12 @@ public boolean checkWriteAccess(Path baseDirectory) { */ return true; } + + /** + * {@inheritDoc} + */ + @Override + public boolean isStorageTypeLocal() { + return storageType.equals(StorageType.LOCAL); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 6fc1d1d2747..4466b4612f7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -154,4 +154,11 @@ public interface IridaFileStorageUtility { * @return if the directory is writable or not */ public boolean checkWriteAccess(Path baseDirectory); + + /** + * Check if the storage type is local + * + * @return if the storage type is local or not + */ + public boolean isStorageTypeLocal(); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index cf37777caf1..6cb5d17ba4c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -2,7 +2,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; import ca.corefacility.bioinformatics.irida.exceptions.UploadException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; @@ -46,7 +45,6 @@ public class AnalysisCollectionServiceGalaxy { private GalaxyHistoriesService galaxyHistoriesService; private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - private StorageType storageType; /** * Builds a new {@link AnalysisCollectionServiceGalaxy} with the given @@ -59,11 +57,10 @@ public class AnalysisCollectionServiceGalaxy { */ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, IridaFileStorageUtility iridaFileStorageUtility, - AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository, StorageType storageType) { + AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { this.galaxyHistoriesService = galaxyHistoriesService; this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; - this.storageType = storageType; } /** @@ -99,7 +96,7 @@ public CollectionResponse uploadSequenceFilesSingleEnd( record to save to the database. The cleanup of these files happens once the final workflow status is set for an analysis. */ - if (!storageType.equals(StorageType.LOCAL)) { + if (!iridaFileStorageUtility.isStorageTypeLocal()) { AnalysisSubmissionTempFile analysisSubmissionTempFile = new AnalysisSubmissionTempFile(submissionId, iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); analysisSubmissionTempFileRepository.save(analysisSubmissionTempFile); @@ -172,7 +169,7 @@ public CollectionResponse uploadSequenceFilesPaired( record to save to the database. The cleanup of these files happens once the final workflow status is set for an analysis. */ - if (!storageType.equals(StorageType.LOCAL)) { + if (!iridaFileStorageUtility.isStorageTypeLocal()) { AnalysisSubmissionTempFile analysisSubmissionTempFileForward = new AnalysisSubmissionTempFile( submissionId, iridaTemporaryFileForward.getFile(), iridaTemporaryFileForward.getDirectoryPath()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index 1a1356eedde..75332b01f15 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -15,7 +15,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; @@ -61,7 +60,6 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisWorkspaceService analysisWorkspaceService; private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - private StorageType storageType; /** * Builds a new AnalysisExecutionScheduledTaskImpl with the given service @@ -83,7 +81,7 @@ public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisS AnalysisExecutionService analysisExecutionServiceGalaxy, CleanupAnalysisSubmissionCondition cleanupCondition, GalaxyJobErrorsService galaxyJobErrorsService, JobErrorRepository jobErrorRepository, EmailController emailController, AnalysisWorkspaceService analysisWorkspaceService, IridaFileStorageUtility iridaFileStorageUtility, - AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository, StorageType storageType) { + AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { this.analysisSubmissionRepository = analysisSubmissionRepository; this.analysisExecutionService = analysisExecutionServiceGalaxy; this.cleanupCondition = cleanupCondition; @@ -93,7 +91,6 @@ public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisS this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisWorkspaceService = analysisWorkspaceService; this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; - this.storageType = storageType; } /** @@ -385,17 +382,15 @@ private void cleanupTemporaryDownloadedFiles(AnalysisSubmission submission) { Cleanup any files that were downloaded from an object store to run an analysis and remove the analysis submission temp file record from the database. */ - if (!storageType.equals(StorageType.LOCAL)) { - List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( - submission.getId()); - logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() - + " temporary files downloaded from object store."); - for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( - new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), - analysisSubmissionTempFile.getFileDirectoryPath())); - analysisSubmissionTempFileRepository.delete(analysisSubmissionTempFile); - } + List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( + submission.getId()); + logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + + " temporary files downloaded from object store."); + for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( + new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), + analysisSubmissionTempFile.getFileDirectoryPath())); + analysisSubmissionTempFileRepository.delete(analysisSubmissionTempFile); } } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java index a577204e034..d6787fad9ea 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java @@ -23,7 +23,6 @@ import com.google.common.collect.Lists; import ca.corefacility.bioinformatics.irida.config.conditions.NonWindowsPlatformCondition; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; @@ -112,8 +111,6 @@ public class AnalysisExecutionServiceTestConfig { @Autowired private IridaFileStorageUtility iridaFileStorageUtility; - @Autowired - private StorageType storageType; @Bean public AnalysisSubmissionSampleProcessor analysisSubmissionSampleProcessor() { @@ -156,7 +153,7 @@ galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy() @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository, storageType); + analysisSubmissionTempFileRepository); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java index 59744fb394f..1f742ae4de1 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java @@ -48,7 +48,6 @@ import ca.corefacility.bioinformatics.irida.config.IridaApiGalaxyTestConfig; import ca.corefacility.bioinformatics.irida.config.conditions.WindowsPlatformCondition; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; @@ -138,8 +137,6 @@ public class SNVPhylAnalysisIT { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Autowired - private StorageType storageType; private Path sequenceFilePathA1; private Path sequenceFilePathA2; @@ -187,7 +184,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.NEVER_CLEANUP, galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository, storageType); + analysisSubmissionTempFileRepository); Path tempDir = Files.createTempDirectory(rootTempDirectory, "snvphylTest"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java index 57d129be4e2..ceaf6597aa1 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java @@ -2,7 +2,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.AnalysisAlreadySetException; import ca.corefacility.bioinformatics.irida.exceptions.PostProcessingException; -import ca.corefacility.bioinformatics.irida.model.IridaClientDetails; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField; import ca.corefacility.bioinformatics.irida.model.sample.Sample; @@ -12,8 +11,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.BioHanselSampleUpdater; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.google.common.collect.ImmutableMap; @@ -40,14 +37,12 @@ public class BioHanselSampleUpdaterTest { private BioHanselSampleUpdater bioHanselSampleUpdater; private MetadataTemplateService metadataTemplateService; private SampleService sampleService; - private IridaFileStorageUtility iridaFileStorageUtility; @Before public void setUp() { metadataTemplateService = mock(MetadataTemplateService.class); sampleService = mock(SampleService.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - bioHanselSampleUpdater = new BioHanselSampleUpdater(metadataTemplateService, sampleService, iridaFileStorageUtility); + bioHanselSampleUpdater = new BioHanselSampleUpdater(metadataTemplateService, sampleService); } @SuppressWarnings("unchecked") diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java index 16d142577fb..4f0e97e7c8c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java @@ -14,7 +14,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.SISTRSampleUpdater; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; @@ -58,9 +57,8 @@ public void setUp() throws IridaWorkflowNotFoundException { iridaWorkflowsService = mock(IridaWorkflowsService.class); iridaWorkflow = mock(IridaWorkflow.class); iridaWorkflowDescription = mock(IridaWorkflowDescription.class); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - updater = new SISTRSampleUpdater(metadataTemplateService, sampleService, iridaWorkflowsService, iridaFileStorageUtility); + updater = new SISTRSampleUpdater(metadataTemplateService, sampleService, iridaWorkflowsService); when(iridaWorkflowsService.getIridaWorkflow(uuid)).thenReturn(iridaWorkflow); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java index 141166ca47e..d9b4b720f56 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java @@ -25,7 +25,6 @@ import ca.corefacility.bioinformatics.irida.config.IridaApiGalaxyTestConfig; import ca.corefacility.bioinformatics.irida.config.conditions.WindowsPlatformCondition; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; @@ -92,8 +91,6 @@ public class GalaxyJobErrorsServiceIT { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Autowired - private StorageType storageType; @Before public void setup() throws URISyntaxException, IOException { @@ -114,7 +111,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository, storageType); + analysisSubmissionTempFileRepository); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java index 622151c5d83..20ce86a1e33 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java @@ -35,7 +35,6 @@ import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowNotFoundException; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -106,9 +105,6 @@ public class AnalysisExecutionScheduledTaskImplIT { @Autowired private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Autowired - private StorageType storageType; - private Path sequenceFilePath; private Path sequenceFilePath2; @@ -134,7 +130,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository, storageType); + analysisSubmissionTempFileRepository); Path sequenceFilePathReal = Paths .get(DatabaseSetupGalaxyITService.class.getResource("testData1.fastq").toURI()); @@ -202,7 +198,7 @@ public void testFullAnalysisRunSuccessNoCleanupAge() throws Exception { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); + iridaFileStorageUtility, analysisSubmissionTempFileRepository); AnalysisSubmission analysisSubmission = analysisExecutionGalaxyITService.setupSubmissionInDatabase(1L, sequenceFilePath, referenceFilePath, validIridaWorkflowId, false); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index d7a51d3e50f..40757a5a44d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -12,7 +12,6 @@ import java.util.UUID; import java.util.concurrent.Future; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import org.joda.time.DateTime; @@ -106,7 +105,6 @@ public class AnalysisExecutionScheduledTaskImplTest { private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - private StorageType storageType; /** * Sets up variables for tests. @@ -115,11 +113,10 @@ public class AnalysisExecutionScheduledTaskImplTest { public void setup() { MockitoAnnotations.initMocks(this); iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); - storageType = StorageType.LOCAL; analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository, storageType); + analysisSubmissionTempFileRepository); analysisSubmission = AnalysisSubmission.builder(workflowId) .name("my analysis") @@ -685,7 +682,7 @@ public void testCleanupAnalysisSubmissionsCompletedOverOneDaySuccess() throws Ex analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); + iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -713,7 +710,7 @@ public void testCleanupAnalysisSubmissionsCompletedCleanupZeroSuccess() throws E analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZERO), galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); + iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -741,7 +738,7 @@ public void testCleanupAnalysisSubmissionsCompletedUnderOneDaySuccess() throws E analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); + iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -769,7 +766,7 @@ public void testCleanupAnalysisSubmissionsCompletedOverUnderOneDaySuccess() thro analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository, storageType); + iridaFileStorageUtility, analysisSubmissionTempFileRepository); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); From c1fb8af8b583d6192f5bbe60e4eac160fa4c5752 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Feb 2021 12:58:49 -0600 Subject: [PATCH 197/655] Updated biohansel and sistr sample updater tests to set the iridaFileStorageUtility for IridaFiles. Refactored storageutility classes to only have one method to get a file size and let IridaFiles return the appropriate response (bytes or human readable string) --- .../IridaFileStorageAwsUtilityImpl.java | 18 ----------------- .../IridaFileStorageAzureUtilityImpl.java | 19 ------------------ .../IridaFileStorageLocalUtilityImpl.java | 20 ------------------- .../filesystem/IridaFileStorageUtility.java | 8 -------- .../AnalysisExecutionScheduledTaskImpl.java | 6 ++++-- .../bioinformatics/irida/util/IridaFiles.java | 9 ++++++++- .../impl/unit/BioHanselSampleUpdaterTest.java | 5 +++++ .../impl/unit/SISTRSampleUpdaterTest.java | 4 ++++ ...nalysisExecutionScheduledTaskImplTest.java | 1 - 9 files changed, 21 insertions(+), 69 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 17aeeac04f9..f25b247657f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -23,7 +23,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -153,23 +152,6 @@ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTempora } } - /** - * {@inheritDoc} - */ - @Override - public String getFileSize(Path file) { - String fileSize = "N/A"; - try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file))) { - fileSize = FileUtils.humanReadableByteCount(s3Object.getObjectMetadata() - .getContentLength(), true); - } catch (AmazonServiceException e) { - logger.error("Unable to get file size from s3 bucket: " + e); - } catch (IOException e) { - logger.error(e.getMessage()); - } - return fileSize; - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index ef5205762f5..96b3cbad347 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -27,7 +27,6 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.util.FileUtils; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; @@ -144,24 +143,6 @@ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTempora } } - /** - * {@inheritDoc} - */ - @Override - public String getFileSize(Path file) { - String fileSize = "N/A"; - try { - // We set the blobClient "path" to which we want to get a file size for - BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - fileSize = FileUtils.humanReadableByteCount(blobClient.getProperties() - .getBlobSize(), true); - } catch (BlobStorageException e) { - logger.error("Couldn't calculate size as the file was not found on azure [" + e + "]"); - } - - return fileSize; - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index c896aa44ef1..f626623a8ae 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -24,7 +24,6 @@ import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import ca.corefacility.bioinformatics.irida.util.FileUtils; /** * Component implementation of file utitlities for local storage @@ -76,25 +75,6 @@ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTempora } } - /** - * {@inheritDoc} - */ - @Override - public String getFileSize(Path file) { - String fileSize = "N/A"; - try { - if (file != null) { - fileSize = FileUtils.humanReadableByteCount(Files.size(file), true); - } - } catch (NoSuchFileException e) { - logger.error("Could not find file " + file); - } catch (IOException e) { - logger.error("Could not calculate file size: ", e); - } - - return fileSize; - } - /** * {@inheritDoc} */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index 4466b4612f7..d52376278f3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -43,14 +43,6 @@ public interface IridaFileStorageUtility { */ public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTemporaryFile); - /** - * Get file size - * - * @param file The {@link Path} to the file - * @return {@link String} size of file retrieved from path - */ - public String getFileSize(Path file); - /** * Write file to storage (azure, aws, or local) * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index 75332b01f15..ded3b5ba567 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -384,8 +384,10 @@ remove the analysis submission temp file record from the database. */ List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( submission.getId()); - logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() - + " temporary files downloaded from object store."); + if (analysisSubmissionTempFiles.size() > 0) { + logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + + " temporary files downloaded from object store."); + } for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 17d6843d44e..3074eeb7d67 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -26,12 +26,19 @@ private IridaFiles() { /** * Gets the file size of the file from the iridaFileStorageUtility + * and returns it as a human readable string * * @param file The path to the file * @return file size as a human readable string */ public static String getFileSize(Path file) { - return iridaFileStorageUtility.getFileSize(file); + String fileSize = "N/A"; + Long fileSizeBytes = iridaFileStorageUtility.getFileSizeBytes(file); + + if(fileSizeBytes > 0){ + fileSize = FileUtils.humanReadableByteCount(fileSizeBytes, true); + } + return fileSize; } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java index ceaf6597aa1..d452ab700f8 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/BioHanselSampleUpdaterTest.java @@ -11,8 +11,12 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.BioHanselSampleUpdater; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -43,6 +47,7 @@ public void setUp() { metadataTemplateService = mock(MetadataTemplateService.class); sampleService = mock(SampleService.class); bioHanselSampleUpdater = new BioHanselSampleUpdater(metadataTemplateService, sampleService); + IridaFiles.setIridaFileStorageUtility(new IridaFileStorageLocalUtilityImpl()); } @SuppressWarnings("unchecked") diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java index 4f0e97e7c8c..a5777187c0d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/unit/SISTRSampleUpdaterTest.java @@ -14,10 +14,13 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.results.updater.impl.SISTRSampleUpdater; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -57,6 +60,7 @@ public void setUp() throws IridaWorkflowNotFoundException { iridaWorkflowsService = mock(IridaWorkflowsService.class); iridaWorkflow = mock(IridaWorkflow.class); iridaWorkflowDescription = mock(IridaWorkflowDescription.class); + IridaFiles.setIridaFileStorageUtility(new IridaFileStorageLocalUtilityImpl()); updater = new SISTRSampleUpdater(metadataTemplateService, sampleService, iridaWorkflowsService); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index 40757a5a44d..4aca399a0dc 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -44,7 +44,6 @@ import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; -import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.impl.AnalysisExecutionScheduledTaskImpl; import ca.corefacility.bioinformatics.irida.service.impl.analysis.submission.CleanupAnalysisSubmissionConditionAge; From 282bab366a17b8646167b7f38e23bf0edfddda3e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Feb 2021 13:33:53 -0600 Subject: [PATCH 198/655] Added mock for analysisSubmissionTempFile to fix broken test --- .../service/impl/AnalysisExecutionScheduledTaskImpl.java | 2 ++ .../impl/unit/AnalysisExecutionScheduledTaskImplTest.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index ded3b5ba567..96713b7516c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -384,10 +384,12 @@ remove the analysis submission temp file record from the database. */ List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( submission.getId()); + if (analysisSubmissionTempFiles.size() > 0) { logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() + " temporary files downloaded from object store."); } + for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index 4aca399a0dc..0e472292668 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -91,6 +91,8 @@ public class AnalysisExecutionScheduledTaskImplTest { @Mock private AnalysisWorkspaceService analysisWorkspaceService; + @Mock + private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; private static final String ANALYSIS_ID = "1"; private static final Long INTERNAL_ID = 1L; @@ -102,8 +104,6 @@ public class AnalysisExecutionScheduledTaskImplTest { private IridaFileStorageUtility iridaFileStorageUtility; - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - /** * Sets up variables for tests. From 28c6030222586ca96886f9096c9e5cde3533bfd1 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Feb 2021 15:52:28 -0600 Subject: [PATCH 199/655] Updated AnalysisSubmissionTempFile class to link to analysis submission via the AnalysisSubmission object rather than just an id --- .../AnalysisSubmissionTempFile.java | 15 +++++++------- .../AnalysisSubmissionTempFileRepository.java | 5 +++-- .../AnalysisCollectionServiceGalaxy.java | 14 ++++++------- .../AnalysisWorkspaceServiceGalaxy.java | 4 ++-- .../AnalysisExecutionScheduledTaskImpl.java | 4 ++-- .../AnalysisCollectionServiceGalaxyIT.java | 19 +++++++++++++----- .../AnalysisWorkspaceServiceGalaxyTest.java | 20 +++++++++---------- .../analysis/AnalysisRepositoryIT.xml | 4 ++++ 8 files changed, 50 insertions(+), 35 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java index 36a330a86a2..6926c757ea4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java @@ -15,13 +15,14 @@ @Entity @Table(name = "analysis_submission_temp_files") public class AnalysisSubmissionTempFile implements IridaThing { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private final Long id; - @NotNull - @Column(name = "analysis_submission_id") - private final Long analysisSubmissionId; + @OneToOne(fetch = FetchType.EAGER, cascade = { CascadeType.MERGE }) + @JoinColumn(name = "analysis_submission_id") + private final AnalysisSubmission analysisSubmission; @NotNull @Column(name = "temp_file_path") @@ -41,7 +42,7 @@ public class AnalysisSubmissionTempFile implements IridaThing { */ @SuppressWarnings("unused") private AnalysisSubmissionTempFile() { - this.analysisSubmissionId = null; + this.analysisSubmission = null; this.filePath = null; this.fileDirectoryPath = null; this.id = null; @@ -52,12 +53,12 @@ private AnalysisSubmissionTempFile() { * Create a new {@link AnalysisSubmissionTempFile} with the given file * analysis submission id, file path, and directory path. * - * @param analysisSubmissionId The id of the {@link AnalysisSubmission} + * @param analysisSubmission The {@link AnalysisSubmission} object * @param filePath The path to the temporary file * @param fileDirectoryPath The path to the temporary file directory */ - public AnalysisSubmissionTempFile(Long analysisSubmissionId, Path filePath, Path fileDirectoryPath) { - this.analysisSubmissionId = analysisSubmissionId; + public AnalysisSubmissionTempFile(AnalysisSubmission analysisSubmission, Path filePath, Path fileDirectoryPath) { + this.analysisSubmission = analysisSubmission; this.filePath = filePath; this.fileDirectoryPath = fileDirectoryPath; this.id = null; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java index 52e8e7a40d7..14095a7db2c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java @@ -4,6 +4,7 @@ import org.springframework.data.jpa.repository.Query; +import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; import ca.corefacility.bioinformatics.irida.repositories.IridaJpaRepository; @@ -19,6 +20,6 @@ public interface AnalysisSubmissionTempFileRepository extends IridaJpaRepository * @param analysisSubmissionId The analysis submission id * @return a list of {@link AnalysisSubmissionTempFile} */ - @Query("FROM AnalysisSubmissionTempFile f WHERE f.analysisSubmissionId = ?1") - List findAllByAnalysisSubmissionId(Long analysisSubmissionId); + @Query("FROM AnalysisSubmissionTempFile f WHERE f.analysisSubmission = ?1") + List findAllByAnalysisSubmission(AnalysisSubmission analysisSubmission); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index 6cb5d17ba4c..b7e05455d73 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -71,7 +71,7 @@ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesSer * {@link SingleEndSequenceFile}. * @param workflowHistory The history to upload the sequence files into. * @param workflowLibrary A temporary library to upload files into. - * @param submissionId The {@link AnalysisSubmission} id + * @param analysisSubmission The {@link AnalysisSubmission} * @return A CollectionResponse for the dataset collection constructed from the * given files. * @throws ExecutionManagerException If there was an error uploading the files. @@ -79,7 +79,7 @@ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesSer */ public CollectionResponse uploadSequenceFilesSingleEnd( Map sampleSequenceFiles, History workflowHistory, - Library workflowLibrary, Long submissionId) throws ExecutionManagerException, IOException { + Library workflowLibrary, AnalysisSubmission analysisSubmission) throws ExecutionManagerException, IOException { CollectionDescription description = new CollectionDescription(); description.setCollectionType(DatasetCollectionType.LIST.toString()); @@ -97,7 +97,7 @@ record to save to the database. The cleanup of these files happens once the fina workflow status is set for an analysis. */ if (!iridaFileStorageUtility.isStorageTypeLocal()) { - AnalysisSubmissionTempFile analysisSubmissionTempFile = new AnalysisSubmissionTempFile(submissionId, + AnalysisSubmissionTempFile analysisSubmissionTempFile = new AnalysisSubmissionTempFile(analysisSubmission, iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); analysisSubmissionTempFileRepository.save(analysisSubmissionTempFile); } @@ -134,7 +134,7 @@ record to save to the database. The cleanup of these files happens once the fina * @param workflowHistory The history to upload the sequence files * into. * @param workflowLibrary A temporary library to upload files into. - * @param submissionId The {@link AnalysisSubmission} id + * @param analysisSubmission The {@link AnalysisSubmission} * @return A CollectionResponse for the dataset collection constructed from the * given files. * @throws ExecutionManagerException If there was an error uploading the files. @@ -142,7 +142,7 @@ record to save to the database. The cleanup of these files happens once the fina */ public CollectionResponse uploadSequenceFilesPaired( Map sampleSequenceFilesPaired, History workflowHistory, - Library workflowLibrary, Long submissionId) throws ExecutionManagerException, IOException { + Library workflowLibrary, AnalysisSubmission analysisSubmission) throws ExecutionManagerException, IOException { CollectionDescription description = new CollectionDescription(); description.setCollectionType(DatasetCollectionType.LIST_PAIRED.toString()); @@ -171,10 +171,10 @@ record to save to the database. The cleanup of these files happens once the fina */ if (!iridaFileStorageUtility.isStorageTypeLocal()) { AnalysisSubmissionTempFile analysisSubmissionTempFileForward = new AnalysisSubmissionTempFile( - submissionId, iridaTemporaryFileForward.getFile(), + analysisSubmission, iridaTemporaryFileForward.getFile(), iridaTemporaryFileForward.getDirectoryPath()); AnalysisSubmissionTempFile analysisSubmissionTempFileReverse = new AnalysisSubmissionTempFile( - submissionId, iridaTemporaryFileReverse.getFile(), + analysisSubmission, iridaTemporaryFileReverse.getFile(), iridaTemporaryFileReverse.getDirectoryPath()); analysisSubmissionTempFileRepository.save(analysisSubmissionTempFileForward); analysisSubmissionTempFileRepository.save(analysisSubmissionTempFileReverse); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java index 18f1bd91a6b..9f44f71a945 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java @@ -220,7 +220,7 @@ public PreparedWorkflowGalaxy prepareAnalysisFiles(AnalysisSubmission analysisSu String workflowSequenceFileSingleInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, sequenceFilesLabelSingle); CollectionResponse collectionResponseSingle = analysisCollectionServiceGalaxy - .uploadSequenceFilesSingleEnd(singleFiles, workflowHistory, workflowLibrary, analysisSubmission.getId()); + .uploadSequenceFilesSingleEnd(singleFiles, workflowHistory, workflowLibrary, analysisSubmission); inputs.setInput(workflowSequenceFileSingleInputId, new WorkflowInvocationInputs.WorkflowInvocationInput( collectionResponseSingle.getId(), WorkflowInvocationInputs.InputSourceType.HDCA)); } @@ -230,7 +230,7 @@ public PreparedWorkflowGalaxy prepareAnalysisFiles(AnalysisSubmission analysisSu String workflowSequenceFilePairedInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, sequenceFilesLabelPaired); CollectionResponse collectionResponsePaired = analysisCollectionServiceGalaxy - .uploadSequenceFilesPaired(pairedFiles, workflowHistory, workflowLibrary, analysisSubmission.getId()); + .uploadSequenceFilesPaired(pairedFiles, workflowHistory, workflowLibrary, analysisSubmission); inputs.setInput(workflowSequenceFilePairedInputId, new WorkflowInvocationInputs.WorkflowInvocationInput( collectionResponsePaired.getId(), WorkflowInvocationInputs.InputSourceType.HDCA)); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index 96713b7516c..0614acfde92 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -382,8 +382,8 @@ private void cleanupTemporaryDownloadedFiles(AnalysisSubmission submission) { Cleanup any files that were downloaded from an object store to run an analysis and remove the analysis submission temp file record from the database. */ - List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmissionId( - submission.getId()); + List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmission( + submission); if (analysisSubmissionTempFiles.size() > 0) { logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java index c96114214bb..b9233220bf7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java @@ -12,9 +12,11 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.DatasetCollectionType; +import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; +import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.galaxy.AnalysisCollectionServiceGalaxy; @@ -90,6 +92,9 @@ public class AnalysisCollectionServiceGalaxyIT { @Autowired private GzipFileProcessor gzipFileProcessor; + @Autowired + private AnalysisSubmissionService analysisSubmissionService; + @Autowired @Qualifier("rootTempDirectory") private Path rootTempDirectory; @@ -123,6 +128,8 @@ public class AnalysisCollectionServiceGalaxyIT { private static final Long ANALYSIS_SUBMISSION_ID = 1L; + private AnalysisSubmission analysisSubmission; + /** * Sets up variables for testing. * @@ -184,6 +191,8 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc pairSequenceFilesCompressedB.add(sequenceFilePathCompressedB); gzipFileProcessor.setDisableFileProcessor(false); + + analysisSubmission = analysisSubmissionService.read(ANALYSIS_SUBMISSION_ID); } /** @@ -286,7 +295,7 @@ public void testUploadSequenceFilesSingleSuccess() throws ExecutionManagerExcept Sample sample1 = sampleRepository.findById(1L).orElse(null); CollectionResponse collectionResponse = analysisCollectionServiceGalaxy - .uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, createdLibrary, ANALYSIS_SUBMISSION_ID); + .uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, createdLibrary, analysisSubmission); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -350,7 +359,7 @@ public void testUploadSequenceFilesSingleCompressedSuccess() throws ExecutionMan sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, - createdLibrary, ANALYSIS_SUBMISSION_ID); + createdLibrary, analysisSubmission); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -398,7 +407,7 @@ public void testUploadSequenceFilesPairedSuccess() throws ExecutionManagerExcept Sample sample1 = sampleRepository.findById(1L).orElse(null); CollectionResponse collectionResponse = analysisCollectionServiceGalaxy - .uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, createdLibrary, ANALYSIS_SUBMISSION_ID); + .uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, createdLibrary, analysisSubmission); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -497,7 +506,7 @@ public void testUploadSequenceFilesPairedCompressedSuccess() throws ExecutionMan sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, - createdLibrary, ANALYSIS_SUBMISSION_ID); + createdLibrary, analysisSubmission); SequenceFilePair pairedSequenceFile = sequenceFiles.iterator().next(); for (SequenceFile file : pairedSequenceFile.getFiles()) { @@ -553,7 +562,7 @@ public void testUploadSequenceFilesPairedFailForward() throws ExecutionManagerEx Map sampleSequenceFilePairs = new HashMap<>(sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, - createdLibrary, ANALYSIS_SUBMISSION_ID); + createdLibrary, analysisSubmission); } private Map historyContentsAsMap(List historyContents) { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index 384e9ba978e..c544dd1004e 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -298,9 +298,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponseSingle); + eq(workflowLibrary), submission)).thenReturn(collectionResponseSingle); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponsePaired); + eq(workflowLibrary), submission)).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -318,9 +318,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class), any(Long.class)); + any(Library.class), submission); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), any(Long.class)); + any(Library.class), submission); } /** @@ -363,7 +363,7 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponseSingle); + eq(workflowLibrary), submission)).thenReturn(collectionResponseSingle); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -380,9 +380,9 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file single entry", workflowInputsMap.containsKey(SEQUENCE_FILE_SINGLE_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class), any(Long.class)); + any(Library.class), submission); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), any(Long.class)); + any(Library.class), submission); } /** @@ -426,7 +426,7 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(Long.class))).thenReturn(collectionResponsePaired); + eq(workflowLibrary), submission)).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -443,9 +443,9 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesSingleEnd(any(Map.class), - any(History.class), any(Library.class), any(Long.class)); + any(History.class), any(Library.class), submission); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), any(Long.class)); + any(Library.class), submission); } /** diff --git a/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml b/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml index 860c29cdf94..212b7353bc3 100644 --- a/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml +++ b/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml @@ -88,5 +88,9 @@ phoneNumber="0000" username="admin" system_role="ROLE_ADMIN" credentialsNonExpired="true" enabled="true" /> + From 11a7a48e89e73f47c7671b3ca0392227d7c3f938 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 19 Feb 2021 16:09:35 -0600 Subject: [PATCH 200/655] Fixed InvalidUseOfMatchers errors --- .../AnalysisWorkspaceServiceGalaxyTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index c544dd1004e..2c68ed8a6b0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -298,9 +298,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), submission)).thenReturn(collectionResponseSingle); + eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponseSingle); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), submission)).thenReturn(collectionResponsePaired); + eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -318,9 +318,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class), submission); + any(Library.class), any(AnalysisSubmission.class)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), submission); + any(Library.class), any(AnalysisSubmission.class)); } /** @@ -363,7 +363,7 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), submission)).thenReturn(collectionResponseSingle); + eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponseSingle); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -380,9 +380,9 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file single entry", workflowInputsMap.containsKey(SEQUENCE_FILE_SINGLE_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class), submission); + any(Library.class), any(AnalysisSubmission.class)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), submission); + any(Library.class), any(AnalysisSubmission.class)); } /** @@ -426,7 +426,7 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), submission)).thenReturn(collectionResponsePaired); + eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -443,9 +443,9 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesSingleEnd(any(Map.class), - any(History.class), any(Library.class), submission); + any(History.class), any(Library.class), any(AnalysisSubmission.class)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), submission); + any(Library.class), any(AnalysisSubmission.class)); } /** From aa1153498d672ff5c1c4588caadfe0dfd97cc74e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 22 Feb 2021 09:02:53 -0600 Subject: [PATCH 201/655] Updated tests and fixed javadoc param --- .../irida/config/services/IridaApiServicesConfig.java | 1 + .../submission/AnalysisSubmissionTempFileRepository.java | 2 +- .../analysis/submission/AnalysisSubmissionRepositoryIT.java | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java index 3590af4543b..bc50191e265 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/IridaApiServicesConfig.java @@ -317,6 +317,7 @@ private String getClasspathResourceBasename(Pattern pattern, Resource x) { * as well as set the implementation in the IridaFiles static class * which uses the this implementation. * + * @param storageType The {@link StorageType} * @return A new {@link IridaFileStorageUtility} implementation. */ @Bean(name = "iridaFileStorageUtility") diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java index 14095a7db2c..a5a1a0e8f46 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java @@ -17,7 +17,7 @@ public interface AnalysisSubmissionTempFileRepository extends IridaJpaRepository /** * Get all {@link AnalysisSubmissionTempFile} objects by submission id. * - * @param analysisSubmissionId The analysis submission id + * @param analysisSubmission The {@link AnalysisSubmission} * @return a list of {@link AnalysisSubmissionTempFile} */ @Query("FROM AnalysisSubmissionTempFile f WHERE f.analysisSubmission = ?1") diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java index 73bbe6477e6..92442f91435 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java @@ -316,7 +316,7 @@ public void testFindBySubmitterSuccess() { Set submissions = analysisSubmissionRepository.findBySubmitter(submitter1); assertNotNull("submissions should not be null", submissions); - assertEquals("there are an invalid number of submissions found", 1, submissions.size()); + assertEquals("there are an invalid number of submissions found", 2, submissions.size()); AnalysisSubmission returnedSubmission = submissions.iterator().next(); assertEquals("the id of the submission returned is incorrect", savedSubmission.getId(), returnedSubmission.getId()); @@ -336,7 +336,7 @@ public void testFindBySubmitterMultipleSuccess() { analysisSubmissionRepository.save(analysisSubmission2); Set submissions = analysisSubmissionRepository.findBySubmitter(submitter1); - assertEquals("there are an invalid number of submissions found", 2, submissions.size()); + assertEquals("there are an invalid number of submissions found", 3, submissions.size()); } /** From fba505cf0963beea296538030c6ee5a29f6aadf0 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 22 Feb 2021 09:31:24 -0600 Subject: [PATCH 202/655] Updated tests --- .../AnalysisSubmissionRepositoryIT.java | 4 +- .../AnalysisCollectionServiceGalaxyIT.java | 46 ++++++++++++++++--- .../analysis/AnalysisRepositoryIT.xml | 5 -- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java index 92442f91435..73bbe6477e6 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionRepositoryIT.java @@ -316,7 +316,7 @@ public void testFindBySubmitterSuccess() { Set submissions = analysisSubmissionRepository.findBySubmitter(submitter1); assertNotNull("submissions should not be null", submissions); - assertEquals("there are an invalid number of submissions found", 2, submissions.size()); + assertEquals("there are an invalid number of submissions found", 1, submissions.size()); AnalysisSubmission returnedSubmission = submissions.iterator().next(); assertEquals("the id of the submission returned is incorrect", savedSubmission.getId(), returnedSubmission.getId()); @@ -336,7 +336,7 @@ public void testFindBySubmitterMultipleSuccess() { analysisSubmissionRepository.save(analysisSubmission2); Set submissions = analysisSubmissionRepository.findBySubmitter(submitter1); - assertEquals("there are an invalid number of submissions found", 3, submissions.size()); + assertEquals("there are an invalid number of submissions found", 2, submissions.size()); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java index b9233220bf7..c92b1feca8c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java @@ -5,18 +5,24 @@ import ca.corefacility.bioinformatics.irida.exceptions.DuplicateSampleException; import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowLoadException; +import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; +import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; +import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; +import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.DatasetCollectionType; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; +import ca.corefacility.bioinformatics.irida.repositories.referencefile.ReferenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; -import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; +import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; +import ca.corefacility.bioinformatics.irida.repositories.user.UserRepository; import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.galaxy.AnalysisCollectionServiceGalaxy; @@ -57,8 +63,7 @@ import java.util.*; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Tests out preparing a workspace for execution of workflows in Galaxy. @@ -93,7 +98,13 @@ public class AnalysisCollectionServiceGalaxyIT { private GzipFileProcessor gzipFileProcessor; @Autowired - private AnalysisSubmissionService analysisSubmissionService; + private SequencingObjectRepository objectRepository; + + @Autowired + private ReferenceFileRepository referenceFileRepository; + + @Autowired + private UserRepository userRepository; @Autowired @Qualifier("rootTempDirectory") @@ -126,9 +137,17 @@ public class AnalysisCollectionServiceGalaxyIT { private static final String FORWARD_NAME = "forward"; private static final String REVERSE_NAME = "reverse"; - private static final Long ANALYSIS_SUBMISSION_ID = 1L; - private AnalysisSubmission analysisSubmission; + private SingleEndSequenceFile singleEndFile; + private SequenceFile sequenceFile; + private UUID workflowId = UUID.randomUUID(); + private ReferenceFile referenceFile; + + private static final String analysisId = "1"; + + private final String analysisName = "analysis 1"; + + private User submitter1; /** * Sets up variables for testing. @@ -192,7 +211,20 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc gzipFileProcessor.setDisableFileProcessor(false); - analysisSubmission = analysisSubmissionService.read(ANALYSIS_SUBMISSION_ID); + submitter1 = userRepository.findById(1L).orElse(null); + + referenceFile = referenceFileRepository.findById(1L).orElse(null); + singleEndFile = (SingleEndSequenceFile) objectRepository.findById(2L).orElse(null); + sequenceFile = singleEndFile.getFileWithId(1L); + assertNotNull(sequenceFile); + Set singleFiles = Sets.newHashSet(singleEndFile); + + analysisSubmission = AnalysisSubmission.builder(workflowId).name(analysisName).inputFiles(singleFiles) + .referenceFile(referenceFile).build(); + analysisSubmission.setRemoteAnalysisId(analysisId); + analysisSubmission.setAnalysisState(AnalysisState.SUBMITTING); + analysisSubmission.setSubmitter(submitter1); + analysisSubmission.setAnalysisCleanedState(AnalysisCleanedState.NOT_CLEANED); } /** diff --git a/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml b/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml index 212b7353bc3..a39a5ebbf86 100644 --- a/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml +++ b/src/test/resources/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisRepositoryIT.xml @@ -88,9 +88,4 @@ phoneNumber="0000" username="admin" system_role="ROLE_ADMIN" credentialsNonExpired="true" enabled="true" /> - - From e6ae78b76abbf6824501d11f8a021efaf02fc644 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 14 Apr 2021 14:10:43 -0500 Subject: [PATCH 203/655] Removed duplicate saving of referencefile. Removed unused import --- .../irida/ria/web/services/UIProjectReferenceFileService.java | 1 - .../bioinformatics/irida/service/impl/ProjectServiceImpl.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java index 87d5eba77b2..529e09cd930 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java @@ -23,7 +23,6 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.references.UIReferenceFile; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java index c74eefe3227..cfc41f6251d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java @@ -588,7 +588,6 @@ public List> getProjectsForSample(Sample sample) { @Transactional @PreAuthorize("hasRole('ROLE_ADMIN') or hasPermission(#project, 'isProjectOwner')") public Join addReferenceFileToProject(Project project, ReferenceFile referenceFile) { - referenceFile = referenceFileRepository.save(referenceFile); ProjectReferenceFileJoin j = new ProjectReferenceFileJoin(project, referenceFile); return prfjRepository.save(j); } From 93b0401897d56947dce8ba37839206c47788bfac Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 14 Apr 2021 14:14:30 -0500 Subject: [PATCH 204/655] Updated to use iridafilestorageutility to get file size bytes as the previous getFileSize method was removed --- .../ria/web/services/UIProjectReferenceFileService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java index 529e09cd930..27ffdd14e7b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectReferenceFileService.java @@ -23,6 +23,7 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.references.UIReferenceFile; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; @@ -77,8 +78,8 @@ public List addReferenceFileToProject(Long projectId, List join = projectService.addReferenceFileToProject(project, referenceFile); ReferenceFile refFile = join.getObject(); - String filesize = iridaFileStorageUtility.getFileSize(refFile.getFile()); - referenceFiles.add(new UIReferenceFile(join, filesize)); + Long filesize = iridaFileStorageUtility.getFileSizeBytes(refFile.getFile()); + referenceFiles.add(new UIReferenceFile(join, FileUtilities.humanReadableByteCount(filesize, true))); } else { referenceFiles.add(new UIReferenceFile(referenceFile)); } From a0deb3bea0e863eea3fb90fdd7e8aaf889d71321 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 15 Apr 2021 08:29:34 -0500 Subject: [PATCH 205/655] Updated test to not check if referenceFileRepository.save is called as the reference file is saved earlier --- .../irida/service/impl/unit/ProjectServiceImplTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/ProjectServiceImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/ProjectServiceImplTest.java index 7602cb431fd..5ad55729bbf 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/ProjectServiceImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/ProjectServiceImplTest.java @@ -414,7 +414,6 @@ public void testAddReferenceFileToProject() throws IOException { projectService.addReferenceFileToProject(p, f); - verify(referenceFileRepository).save(f); verify(prfjRepository).save(new ProjectReferenceFileJoin(p, f)); } From a697977e283202587cde4ae8ed67c05feb6577c5 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 19 Apr 2021 09:42:10 -0500 Subject: [PATCH 206/655] Readded accidentally removed getStorageType method --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 8 ++++++++ .../filesystem/IridaFileStorageAzureUtilityImpl.java | 8 ++++++++ .../filesystem/IridaFileStorageLocalUtilityImpl.java | 8 ++++++++ .../repositories/filesystem/IridaFileStorageUtility.java | 7 +++++++ 4 files changed, 31 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index f25b247657f..e08af40c6af 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -403,4 +403,12 @@ public boolean checkWriteAccess(Path baseDirectory) { public boolean isStorageTypeLocal() { return storageType.equals(StorageType.LOCAL); } + + /** + * {@inheritDoc} + */ + @Override + public String getStorageType() { + return storageType.toString(); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 96b3cbad347..9b742f3d7f2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -399,4 +399,12 @@ public boolean checkWriteAccess(Path baseDirectory) { public boolean isStorageTypeLocal() { return storageType.equals(StorageType.LOCAL); } + + /** + * {@inheritDoc} + */ + @Override + public String getStorageType() { + return storageType.toString(); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index f626623a8ae..82ea0188524 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -292,4 +292,12 @@ public boolean checkWriteAccess(Path baseDirectory) { public boolean isStorageTypeLocal() { return storageType.equals(StorageType.LOCAL); } + + /** + * {@inheritDoc} + */ + @Override + public String getStorageType() { + return storageType.toString(); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java index d52376278f3..c4b8c2cd008 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageUtility.java @@ -153,4 +153,11 @@ public interface IridaFileStorageUtility { * @return if the storage type is local or not */ public boolean isStorageTypeLocal(); + + /** + * Get the storage type. + * + * @return {@link String} The storage type + */ + public String getStorageType(); } From 7085dc2c26838bde37d6f3330d78d5565c0da639 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 20 Apr 2021 10:05:43 -0500 Subject: [PATCH 207/655] Updated to set datastorage to remote if using an object store otherwisse defaults to local --- .../workspace/galaxy/AnalysisCollectionServiceGalaxy.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index b7e05455d73..9a71167451a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -45,6 +45,7 @@ public class AnalysisCollectionServiceGalaxy { private GalaxyHistoriesService galaxyHistoriesService; private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; + private DataStorage dataStorageType; /** * Builds a new {@link AnalysisCollectionServiceGalaxy} with the given @@ -61,6 +62,7 @@ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesSer this.galaxyHistoriesService = galaxyHistoriesService; this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; + this.dataStorageType = iridaFileStorageUtility.isStorageTypeLocal() ? DataStorage.LOCAL : DataStorage.REMOTE; } /** @@ -105,7 +107,7 @@ record to save to the database. The cleanup of these files happens once the fina // upload files to library and then to a history Map pathHistoryDatasetId = galaxyHistoriesService.filesToLibraryToHistory(samplesMap.keySet(), - workflowHistory, workflowLibrary, DataStorage.LOCAL); + workflowHistory, workflowLibrary, dataStorageType); for (Path sequenceFilePath : samplesMap.keySet()) { if (!pathHistoryDatasetId.containsKey(sequenceFilePath)) { @@ -183,7 +185,7 @@ record to save to the database. The cleanup of these files happens once the fina // upload files to library and then to a history Map pathHistoryDatasetId = galaxyHistoriesService.filesToLibraryToHistory(pathsToUpload, - workflowHistory, workflowLibrary, DataStorage.LOCAL); + workflowHistory, workflowLibrary, dataStorageType); for (Sample sample : sampleSequenceFilesPaired.keySet()) { Path fileForward = samplesMapPairForward.get(sample); From e61b5fdf870aaae9cc9c7adab764a6bfffcc9e46 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 26 Apr 2021 13:38:11 -0500 Subject: [PATCH 208/655] Updated to remove permissions required for galaxy to read files as we now just upload the files to galaxy if using a remote filesystem --- .../upload/galaxy/GalaxyLibrariesService.java | 51 +++++++++++++------ .../IridaFileStorageAwsUtilityImpl.java | 6 --- .../IridaFileStorageAzureUtilityImpl.java | 6 --- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index 63ae0aefc5d..8a01462902b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -151,21 +151,42 @@ public String fileToLibrary(Path path, InputFileType fileType, File file = path.toFile(); try { - LibraryContent rootContent = librariesClient.getRootFolder(library - .getId()); - FilesystemPathsLibraryUpload upload = new FilesystemPathsLibraryUpload(); - upload.setFolderId(rootContent.getId()); - - upload.setContent(file.getAbsolutePath()); - upload.setName(file.getName()); - upload.setLinkData(DataStorage.LOCAL.equals(dataStorage)); - upload.setFileType(fileType.toString()); - - GalaxyObject uploadObject = librariesClient.uploadFilesystemPaths( - library.getId(), upload); - - - return uploadObject.getId(); + LibraryContent rootContent = librariesClient.getRootFolder(library.getId()); + if(dataStorage.equals(DataStorage.LOCAL)) { + FilesystemPathsLibraryUpload upload = new FilesystemPathsLibraryUpload(); + upload.setFolderId(rootContent.getId()); + + upload.setContent(file.getAbsolutePath()); + upload.setName(file.getName()); + upload.setLinkData(true); + upload.setFileType(fileType.toString()); + + GalaxyObject uploadObject = librariesClient.uploadFilesystemPaths(library.getId(), upload); + return uploadObject.getId(); + } else { + // If the dataStorage isn't local we upload the file to the library instead of linking + FileLibraryUpload fileLibraryUpload = new FileLibraryUpload(); + fileLibraryUpload.setContent(file.getAbsolutePath()); + fileLibraryUpload.setName(file.getName()); + fileLibraryUpload.setFolderId(rootContent.getId()); + fileLibraryUpload.setFile(path.toFile()); + fileLibraryUpload.setFileType(fileType.toString()); + + ClientResponse clientResponse = librariesClient.uploadFile(library.getId(), fileLibraryUpload); + + if (clientResponse == null) { + throw new UploadException( + "Could not upload " + file + " to library " + library.getId() + " ClientResponse is null"); + } else if (ClientResponse.Status.OK.getStatusCode() != clientResponse.getStatusInfo() + .getStatusCode()) { + String message = "Could not upload " + file + " to library " + library.getId() + " ClientResponse: " + + clientResponse.getStatusInfo() + " " + clientResponse.getEntity(String.class); + logger.error(message); + throw new UploadException(message); + } else { + return library.getId(); + } + } } catch (RuntimeException e) { throw new UploadException(e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index f25b247657f..2c923603f03 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -62,14 +62,11 @@ public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, St */ @Override public IridaTemporaryFile getTemporaryFile(Path file) { - String perm = "rwxrwxr-x"; try { logger.trace("Getting file from aws s3 [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory("aws-tmp-"); Path tempFile = tempDirectory.resolve(file.getFileName() .toString()); - Set permissions = PosixFilePermissions.fromString(perm); - Files.setPosixFilePermissions(tempDirectory, permissions); try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { @@ -94,14 +91,11 @@ public IridaTemporaryFile getTemporaryFile(Path file) { */ @Override public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { - String perm = "rwxrwxr-x"; try { logger.trace("Getting file from aws s3 [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory(prefix + "-aws-tmp-"); Path tempFile = tempDirectory.resolve(file.getFileName() .toString()); - Set permissions = PosixFilePermissions.fromString(perm); - Files.setPosixFilePermissions(tempDirectory, permissions); try (S3Object s3Object = s3.getObject(bucketName, getAwsFileAbsolutePath(file)); S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 96b3cbad347..d8d4dbaffec 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -61,7 +61,6 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St */ @Override public IridaTemporaryFile getTemporaryFile(Path file) { - String perm = "rwxrwxr-x"; try { // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); @@ -71,8 +70,6 @@ public IridaTemporaryFile getTemporaryFile(Path file) { Path tempFile = tempDirectory.resolve(file.getFileName() .toString()); org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); - Set permissions = PosixFilePermissions.fromString(perm); - Files.setPosixFilePermissions(tempDirectory, permissions); return new IridaTemporaryFile(tempFile, tempDirectory); } catch (IOException e) { logger.error(e.getMessage()); @@ -89,7 +86,6 @@ public IridaTemporaryFile getTemporaryFile(Path file) { */ @Override public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { - String perm = "rwxrwxr-x"; try { // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); @@ -99,8 +95,6 @@ public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { Path tempFile = tempDirectory.resolve(file.getFileName() .toString()); org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); - Set permissions = PosixFilePermissions.fromString(perm); - Files.setPosixFilePermissions(tempDirectory, permissions); return new IridaTemporaryFile(tempFile, tempDirectory); } catch (IOException e) { logger.error(e.getMessage()); From 70b13b01fe1850cef296b4667e1cd895384902a8 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 26 Apr 2021 14:12:51 -0500 Subject: [PATCH 209/655] Updated setting of file size for reference file --- .../irida/ria/web/ajax/dto/references/UIReferenceFile.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java index 33d74cee958..7fc30f6ec4e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java @@ -38,8 +38,7 @@ public UIReferenceFile(ReferenceFile file) throws IOException { this.projectId = null; this.projectName = null; this.createdDate = file.getCreatedDate(); - Path path = file.getFile(); - this.size = FileUtilities.humanReadableByteCount(Files.size(path), true); + this.size = file.getFileSize(); } public Long getId() { From d6922e651aee002a44e04b3345afe878907be435 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 26 Apr 2021 14:24:19 -0500 Subject: [PATCH 210/655] Removed unused imports --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 3 --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 3 --- .../irida/ria/web/ajax/dto/references/UIReferenceFile.java | 3 --- 3 files changed, 9 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 2c923603f03..1bb951a2ab8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -5,11 +5,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index d8d4dbaffec..fb65c5f69dd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -10,11 +10,8 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.zip.GZIPInputStream; import org.slf4j.Logger; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java index 7fc30f6ec4e..67d15d37b7a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/references/UIReferenceFile.java @@ -1,14 +1,11 @@ package ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.references; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Date; import ca.corefacility.bioinformatics.irida.model.joins.Join; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; -import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; /** * Represents a reference file in the UI. From 4bd005c5341b6e0f521e6b935d11986abbf28be0 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 27 Apr 2021 14:29:31 -0500 Subject: [PATCH 211/655] Removed iridaFileStorageUtility autowiring from GalaxyHistoriesService and moved into AnalysisWorkspaceServiceGalaxy when uploading reference files. Updated azure iridafilestorageutility to upload files in blocks rather than in one go to prevent timeouts with larger files --- .../AnalysisExecutionServiceConfig.java | 2 +- .../analysis/ExecutionManagerConfig.java | 5 +---- .../upload/galaxy/GalaxyHistoriesService.java | 11 +++-------- .../IridaFileStorageAzureUtilityImpl.java | 18 +++++++++++------- .../galaxy/AnalysisWorkspaceServiceGalaxy.java | 14 ++++++++++++-- .../config/IridaApiNoGalaxyTestConfig.java | 7 +------ .../AnalysisExecutionServiceTestConfig.java | 2 +- .../analysis/GalaxyExecutionTestConfig.java | 5 +---- .../integration/GalaxyHistoriesServiceIT.java | 6 +----- .../galaxy/integration/GalaxyWorkflowsIT.java | 6 +----- .../unit/GalaxyHistoriesServiceTest.java | 6 +----- .../AnalysisWorkspaceServiceGalaxyIT.java | 7 +------ .../AnalysisWorkspaceServiceGalaxyTest.java | 8 +++++++- 13 files changed, 42 insertions(+), 55 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java index c52f1b0b14a..689d46ca458 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java @@ -168,7 +168,7 @@ public AnalysisWorkspaceServiceGalaxy analysisWorkspaceService() { return new AnalysisWorkspaceServiceGalaxy(galaxyHistoriesService, galaxyWorkflowService, galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy(), analysisProvenanceService(), analysisParameterServiceGalaxy, - sequencingObjectService); + sequencingObjectService, iridaFileStorageUtility); } @Lazy diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java index 29ea949f0c5..a6918c3f840 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/ExecutionManagerConfig.java @@ -29,7 +29,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; import com.github.jmchilton.blend4j.galaxy.GalaxyInstanceFactory; @@ -91,8 +90,6 @@ public class ExecutionManagerConfig { @Autowired private Validator validator; - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; /** * Builds a new ExecutionManagerGalaxy from the given properties. @@ -279,7 +276,7 @@ public LibrariesClient librariesClient() throws ExecutionManagerConfigurationExc @Lazy @Bean public GalaxyHistoriesService galaxyHistoriesService() throws ExecutionManagerConfigurationException { - return new GalaxyHistoriesService(historiesClient(), toolsClient(), galaxyLibrariesService(), iridaFileStorageUtility); + return new GalaxyHistoriesService(historiesClient(), toolsClient(), galaxyLibrariesService()); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java index 9aa9f64e7c0..5f3b8398e11 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java @@ -66,17 +66,15 @@ public class GalaxyHistoriesService { private GalaxyLibrariesService librariesService; - private IridaFileStorageUtility iridaFileStorageUtility; /** * Builds a new GalaxyHistory object for working with Galaxy Histories. * @param historiesClient The HistoriesClient for interacting with Galaxy histories. * @param toolsClient The ToolsClient for interacting with tools in Galaxy. * @param librariesService A service for dealing with Galaxy libraries. - * @param iridaFileStorageUtility The file storage implementation */ public GalaxyHistoriesService(HistoriesClient historiesClient, - ToolsClient toolsClient, GalaxyLibrariesService librariesService, IridaFileStorageUtility iridaFileStorageUtility) { + ToolsClient toolsClient, GalaxyLibrariesService librariesService) { checkNotNull(historiesClient, "historiesClient is null"); checkNotNull(toolsClient, "toolsClient is null"); checkNotNull(librariesService, "librariesService is null"); @@ -84,7 +82,6 @@ public GalaxyHistoriesService(HistoriesClient historiesClient, this.historiesClient = historiesClient; this.toolsClient = toolsClient; this.librariesService = librariesService; - this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -154,10 +151,9 @@ public Dataset fileToHistory(Path path, InputFileType fileType, History history) checkNotNull(fileType, "fileType is null"); checkNotNull(history, "history is null"); checkNotNull(history.getId(), "history id is null"); - checkState(iridaFileStorageUtility.fileExists(path), "path " + path + " does not exist"); + checkState(path.toFile().exists(), "path " + path + " does not exist"); - IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(path); - File file = iridaTemporaryFile.getFile().toFile(); + File file = path.toFile(); FileUploadRequest uploadRequest = new FileUploadRequest(history.getId(), file); uploadRequest.setFileType(fileType.toString()); @@ -177,7 +173,6 @@ public Dataset fileToHistory(Path path, InputFileType fileType, History history) throw new UploadException(message); } else { Dataset dataset = getDatasetForFileInHistory(file.getName(), history.getId()); - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); return dataset; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index fb65c5f69dd..980124e4f9f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -10,6 +10,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.zip.GZIPInputStream; @@ -25,12 +26,8 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobRange; -import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.*; +import com.azure.storage.blob.models.*; import com.azure.storage.blob.specialized.BlobInputStream; /** @@ -61,6 +58,7 @@ public IridaTemporaryFile getTemporaryFile(Path file) { try { // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); + try (InputStream initialStream = blobClient.openInputStream()) { logger.trace("Getting file from azure [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory("azure-tmp-"); @@ -143,7 +141,13 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(target)); try { logger.trace("Uploading file to azure: [" + target.getFileName() + "]"); - blobClient.uploadFromFile(source.toString(), false); + + // Upload the file in blocks rather than all at once to prevent a timeout if the file is large. + int blockSize = 2 * 1024 * 1024; //2MB + ParallelTransferOptions parallelTransferOptions = new ParallelTransferOptions(blockSize, 8, null); + blobClient.uploadFromFile(source.toString(), parallelTransferOptions, new BlobHttpHeaders(), null, AccessTier.HOT, + new BlobRequestConditions(), Duration.ofMinutes(10)); + logger.trace("File uploaded to: [" + blobClient.getBlobUrl() + "]"); } catch (BlobStorageException e) { logger.error("Unable to upload file to azure [" + e + "]"); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java index 9f44f71a945..2d5ef548e5f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java @@ -23,6 +23,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; @@ -69,6 +71,8 @@ public class AnalysisWorkspaceServiceGalaxy implements AnalysisWorkspaceService private SequencingObjectService sequencingObjectService; + private IridaFileStorageUtility iridaFileStorageUtility; + /** * Builds a new {@link AnalysisWorkspaceServiceGalaxy} with the given * information. @@ -82,6 +86,7 @@ public class AnalysisWorkspaceServiceGalaxy implements AnalysisWorkspaceService * @param analysisProvenanceServiceGalaxy The service for provenance information. * @param analysisParameterServiceGalaxy A service for setting up parameters in Galaxy. * @param sequencingObjectService A service for reading {@link SequencingObject}s + * @param iridaFileStorageUtility The file storage implementation */ public AnalysisWorkspaceServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, GalaxyWorkflowService galaxyWorkflowService, GalaxyLibrariesService galaxyLibrariesService, @@ -89,7 +94,7 @@ public AnalysisWorkspaceServiceGalaxy(GalaxyHistoriesService galaxyHistoriesServ AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy, AnalysisProvenanceServiceGalaxy analysisProvenanceServiceGalaxy, AnalysisParameterServiceGalaxy analysisParameterServiceGalaxy, - SequencingObjectService sequencingObjectService) { + SequencingObjectService sequencingObjectService, IridaFileStorageUtility iridaFileStorageUtility) { this.galaxyHistoriesService = galaxyHistoriesService; this.galaxyWorkflowService = galaxyWorkflowService; this.galaxyLibrariesService = galaxyLibrariesService; @@ -98,6 +103,7 @@ public AnalysisWorkspaceServiceGalaxy(GalaxyHistoriesService galaxyHistoriesServ this.analysisProvenanceServiceGalaxy = analysisProvenanceServiceGalaxy; this.analysisParameterServiceGalaxy = analysisParameterServiceGalaxy; this.sequencingObjectService = sequencingObjectService; + this.iridaFileStorageUtility = iridaFileStorageUtility; } /** @@ -295,7 +301,9 @@ private void prepareReferenceFile(ReferenceFile referenceFile, History workflowH WorkflowDetails workflowDetails, WorkflowInvocationInputs inputs) throws UploadException, GalaxyDatasetException, WorkflowException { - Dataset referenceDataset = galaxyHistoriesService.fileToHistory(referenceFile.getFile(), InputFileType.FASTA, + IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(referenceFile.getFile()); + + Dataset referenceDataset = galaxyHistoriesService.fileToHistory(iridaTemporaryFile.getFile(), InputFileType.FASTA, workflowHistory); String workflowReferenceFileInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, @@ -303,6 +311,8 @@ private void prepareReferenceFile(ReferenceFile referenceFile, History workflowH inputs.setInput(workflowReferenceFileInputId, new WorkflowInvocationInputs.WorkflowInvocationInput(referenceDataset.getId(), WorkflowInvocationInputs.InputSourceType.HDA)); + + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java index 0975ba17c04..f6dbe7ada35 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/IridaApiNoGalaxyTestConfig.java @@ -8,7 +8,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -21,7 +20,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.analysis.execution.galaxy.AnalysisExecutionServiceGalaxyCleanupAsync; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; @@ -65,9 +63,6 @@ public class IridaApiNoGalaxyTestConfig { private static final Logger logger = LoggerFactory.getLogger(IridaApiNoGalaxyTestConfig.class); - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - /** * @return An ExecutorService executing code in the same thread for testing * purposes. @@ -93,7 +88,7 @@ public Future cleanupSubmission(AnalysisSubmission analysisS @Bean public GalaxyHistoriesService galaxyHistoriesService(HistoriesClient historiesClient, ToolsClient toolsClient, GalaxyLibrariesService librariesService) { - return new GalaxyHistoriesService(historiesClient, toolsClient, librariesService, iridaFileStorageUtility); + return new GalaxyHistoriesService(historiesClient, toolsClient, librariesService); } @Bean diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java index d6787fad9ea..0c7319f5015 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java @@ -146,7 +146,7 @@ public AnalysisWorkspaceServiceGalaxy analysisWorkspaceService() { return new AnalysisWorkspaceServiceGalaxy(galaxyHistoriesService, galaxyWorkflowService, galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy(), analysisProvenanceServiceGalaxy(), analysisParameterServiceGalaxy, - sequencingObjectService); + sequencingObjectService, iridaFileStorageUtility); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java index bdb23329542..49ab1dd71d4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/GalaxyExecutionTestConfig.java @@ -15,7 +15,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.github.jmchilton.blend4j.galaxy.*; @@ -30,8 +29,6 @@ public class GalaxyExecutionTestConfig { @Autowired private LocalGalaxy localGalaxy; - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; /** * Timeout in seconds to stop polling a Galaxy library. @@ -55,7 +52,7 @@ public GalaxyInstance galaxyInstance() { public GalaxyHistoriesService galaxyHistoriesService() { HistoriesClient historiesClient = localGalaxy.getGalaxyInstanceAdmin().getHistoriesClient(); ToolsClient toolsClient = localGalaxy.getGalaxyInstanceAdmin().getToolsClient(); - return new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService(), iridaFileStorageUtility); + return new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService()); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java index 72e086eb282..08320f9c7c7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java @@ -43,8 +43,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; import com.github.jmchilton.blend4j.galaxy.GalaxyResponseException; @@ -85,7 +83,6 @@ public class GalaxyHistoriesServiceIT { private GalaxyInstance galaxyInstanceAdmin; private GalaxyLibrariesService galaxyLibrariesService; private HistoriesClient historiesClient; - private IridaFileStorageUtility iridaFileStorageUtility; private Path dataFile; private Path dataFile2; @@ -123,12 +120,11 @@ public void setup() throws URISyntaxException, IOException, CreateLibraryExcepti historiesClient = galaxyInstanceAdmin.getHistoriesClient(); ToolsClient toolsClient = galaxyInstanceAdmin.getToolsClient(); LibrariesClient librariesClient = galaxyInstanceAdmin.getLibrariesClient(); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, - galaxyLibrariesService, iridaFileStorageUtility); + galaxyLibrariesService); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java index 7dea6716d0d..e37838b8327 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyWorkflowsIT.java @@ -47,8 +47,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.github.jmchilton.blend4j.galaxy.beans.*; import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; @@ -96,7 +94,6 @@ public class GalaxyWorkflowsIT { private LibrariesClient librariesClient; private GalaxyWorkflowService galaxyWorkflowService; private GalaxyHistoriesService galaxyHistory; - private IridaFileStorageUtility iridaFileStorageUtility; private static final InputFileType FILE_TYPE = InputFileType.FASTQ_SANGER; private static final InputFileType INVALID_FILE_TYPE = null; @@ -156,10 +153,9 @@ public void setup() throws URISyntaxException, IOException { workflowsClient = galaxyAdminInstance.getWorkflowsClient(); historiesClient = galaxyAdminInstance.getHistoriesClient(); librariesClient = galaxyAdminInstance.getLibrariesClient(); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); - galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageUtility); + galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); galaxyWorkflowService = new GalaxyWorkflowService(workflowsClient, StandardCharsets.UTF_8); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java index b29ec035600..bdda36606d8 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/unit/GalaxyHistoriesServiceTest.java @@ -33,8 +33,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.Util; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.github.jmchilton.blend4j.galaxy.ToolsClient; @@ -63,7 +61,6 @@ public class GalaxyHistoriesServiceTest { @Mock private ClientResponse invalidResponse; @Mock private ClientResponse okayResponse; @Mock private GalaxyLibrariesService galaxyLibrariesService; - private IridaFileStorageUtility iridaFileStorageUtility; private GalaxyHistoriesService galaxyHistory; @@ -100,10 +97,9 @@ public void setup() throws URISyntaxException { ClientResponse.Status.OK); when(invalidResponse.getStatusInfo()).thenReturn( ClientResponse.Status.FORBIDDEN); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); galaxyHistory = new GalaxyHistoriesService(historiesClient, toolsClient, - galaxyLibrariesService, iridaFileStorageUtility); + galaxyLibrariesService); dataFile = Paths.get(this.getClass().getResource("testData1.fastq").toURI()); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java index 90adc379f55..461f85e219f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisWorkspaceServiceGalaxyIT.java @@ -82,8 +82,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.Util; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; import ca.corefacility.bioinformatics.irida.service.DatabaseSetupGalaxyITService; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; @@ -186,8 +184,6 @@ public class AnalysisWorkspaceServiceGalaxyIT { private static final String SAMPLE1_NAME = "sample1"; - private IridaFileStorageUtility iridaFileStorageUtility; - /** * Sets up variables for testing. * @@ -231,11 +227,10 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc HistoriesClient historiesClient = galaxyInstanceAdmin.getHistoriesClient(); ToolsClient toolsClient = galaxyInstanceAdmin.getToolsClient(); LibrariesClient librariesClient = galaxyInstanceAdmin.getLibrariesClient(); - iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); GalaxyLibrariesService galaxyLibrariesService = new GalaxyLibrariesService(librariesClient, LIBRARY_POLLING_TIME, LIBRARY_TIMEOUT, 1); - galaxyHistoriesService = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService, iridaFileStorageUtility); + galaxyHistoriesService = new GalaxyHistoriesService(historiesClient, toolsClient, galaxyLibrariesService); pairSequenceFiles1A = new ArrayList<>(); pairSequenceFiles1A.add(sequenceFilePathA); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index 2c68ed8a6b0..99502401efd 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -20,6 +20,8 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyLibrariesService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequenceFileRepository; import ca.corefacility.bioinformatics.irida.service.SequencingObjectService; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.galaxy.AnalysisCollectionServiceGalaxy; @@ -87,6 +89,8 @@ public class AnalysisWorkspaceServiceGalaxyTest { private AnalysisWorkspaceServiceGalaxy workflowPreparation; + private IridaFileStorageUtility iridaFileStorageUtility; + private Set inputFiles; private ReferenceFile referenceFile; private Path refFile; @@ -196,10 +200,12 @@ public void setup() throws IOException, UploadException, GalaxyDatasetException workflowDetails = new WorkflowDetails(); workflowDetails.setId(WORKFLOW_ID); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + workflowPreparation = new AnalysisWorkspaceServiceGalaxy(galaxyHistoriesService, galaxyWorkflowService, galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy, analysisProvenanceServiceGalaxy, analysisParameterServiceGalaxy, - sequencingObjectService); + sequencingObjectService, iridaFileStorageUtility); output1Dataset = new Dataset(); output1Dataset.setId("1"); From 33da2640d5c18f6b6cf8ae5c7199e1906dd5bc91 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 27 Apr 2021 14:47:11 -0500 Subject: [PATCH 212/655] Updated azure file storage utility to download files directly rather than through a stream when getting files from azure --- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 980124e4f9f..57774b2e981 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -59,12 +59,12 @@ public IridaTemporaryFile getTemporaryFile(Path file) { // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - try (InputStream initialStream = blobClient.openInputStream()) { + try { logger.trace("Getting file from azure [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory("azure-tmp-"); Path tempFile = tempDirectory.resolve(file.getFileName() .toString()); - org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); + blobClient.downloadToFile(tempFile.toString()); return new IridaTemporaryFile(tempFile, tempDirectory); } catch (IOException e) { logger.error(e.getMessage()); @@ -84,12 +84,12 @@ public IridaTemporaryFile getTemporaryFile(Path file, String prefix) { try { // We set the blobClient "path" to which file we want to get BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file)); - try (InputStream initialStream = blobClient.openInputStream()) { + try { logger.trace("Getting file from azure [" + file.toString() + "]"); Path tempDirectory = Files.createTempDirectory(prefix + "-azure-tmp-"); Path tempFile = tempDirectory.resolve(file.getFileName() .toString()); - org.apache.commons.io.FileUtils.copyInputStreamToFile(initialStream, tempFile.toFile()); + blobClient.downloadToFile(tempFile.toString()); return new IridaTemporaryFile(tempFile, tempDirectory); } catch (IOException e) { logger.error(e.getMessage()); From d6a11c24c31816b5a0436ff2cbc3acab5c0ae65d Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 28 Apr 2021 08:53:59 -0500 Subject: [PATCH 213/655] Removed analysisSubmissionTempFile repository and table as they are no longer required since when using an object store once the files are uploaded to galaxy we remove them --- .../AnalysisExecutionServiceConfig.java | 7 +- .../AnalysisScheduledTaskConfig.java | 10 +- .../AnalysisSubmissionTempFile.java | 96 ------------------- .../upload/galaxy/GalaxyHistoriesService.java | 2 - .../AnalysisSubmissionTempFileRepository.java | 25 ----- .../AnalysisCollectionServiceGalaxy.java | 63 ++++-------- .../AnalysisWorkspaceServiceGalaxy.java | 4 +- .../AnalysisExecutionScheduledTaskImpl.java | 46 +-------- .../database/changesets/21.01/all-changes.xml | 2 - .../analysis-submission-cloud-temp-files.xml | 32 ------- .../AnalysisExecutionServiceTestConfig.java | 7 +- .../integration/SNVPhylAnalysisIT.java | 12 +-- .../integration/GalaxyJobErrorsServiceIT.java | 11 +-- .../AnalysisCollectionServiceGalaxyIT.java | 19 +--- .../AnalysisWorkspaceServiceGalaxyTest.java | 20 ++-- .../AnalysisExecutionScheduledTaskImplIT.java | 14 +-- ...nalysisExecutionScheduledTaskImplTest.java | 30 +----- 17 files changed, 50 insertions(+), 350 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java delete mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java index 689d46ca458..79bf8fddebf 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceConfig.java @@ -25,7 +25,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; import ca.corefacility.bioinformatics.irida.plugins.IridaPlugin; import ca.corefacility.bioinformatics.irida.plugins.IridaPluginException; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; import ca.corefacility.bioinformatics.irida.service.AnalysisService; @@ -107,9 +106,6 @@ public class AnalysisExecutionServiceConfig { @Autowired private List defaultAnalysisSampleUpdaters; - @Autowired - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Autowired private IridaFileStorageUtility iridaFileStorageUtility; @@ -180,7 +176,6 @@ public AnalysisProvenanceServiceGalaxy analysisProvenanceService() { @Lazy @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { - return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository); + return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java index 3e1abb7ed83..8759a3525fb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/services/scheduled/AnalysisScheduledTaskConfig.java @@ -2,9 +2,7 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; @@ -52,12 +50,6 @@ public class AnalysisScheduledTaskConfig { @Autowired private AnalysisWorkspaceService analysisWorkspaceService; - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - - @Autowired - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - /** * Defines the time to clean up in number of days a submission must exist before it is cleaned up. */ @@ -133,7 +125,7 @@ public void cleanupAnalysisSubmissions() { public AnalysisExecutionScheduledTask analysisExecutionScheduledTask() { return new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, cleanupAnalysisSubmissionCondition(), galaxyJobErrorsService, jobErrorRepository, emailController, - analysisWorkspaceService, iridaFileStorageUtility, analysisSubmissionTempFileRepository); + analysisWorkspaceService); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java deleted file mode 100644 index 6926c757ea4..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/submission/AnalysisSubmissionTempFile.java +++ /dev/null @@ -1,96 +0,0 @@ -package ca.corefacility.bioinformatics.irida.model.workflow.submission; - -import java.nio.file.Path; -import java.util.Date; - -import javax.persistence.*; -import javax.validation.constraints.NotNull; - -import ca.corefacility.bioinformatics.irida.model.IridaThing; - -/** - * A temporary file which required by an {@link AnalysisSubmission} when - * the storage type is an object store. - */ -@Entity -@Table(name = "analysis_submission_temp_files") -public class AnalysisSubmissionTempFile implements IridaThing { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private final Long id; - - @OneToOne(fetch = FetchType.EAGER, cascade = { CascadeType.MERGE }) - @JoinColumn(name = "analysis_submission_id") - private final AnalysisSubmission analysisSubmission; - - @NotNull - @Column(name = "temp_file_path") - private final Path filePath; - - @NotNull - @Column(name = "temp_file_directory_path") - private final Path fileDirectoryPath; - - @NotNull - @Temporal(TemporalType.TIMESTAMP) - @Column(name = "created_date") - private final Date createdDate; - - /** - * for hibernate - */ - @SuppressWarnings("unused") - private AnalysisSubmissionTempFile() { - this.analysisSubmission = null; - this.filePath = null; - this.fileDirectoryPath = null; - this.id = null; - this.createdDate = null; - } - - /** - * Create a new {@link AnalysisSubmissionTempFile} with the given file - * analysis submission id, file path, and directory path. - * - * @param analysisSubmission The {@link AnalysisSubmission} object - * @param filePath The path to the temporary file - * @param fileDirectoryPath The path to the temporary file directory - */ - public AnalysisSubmissionTempFile(AnalysisSubmission analysisSubmission, Path filePath, Path fileDirectoryPath) { - this.analysisSubmission = analysisSubmission; - this.filePath = filePath; - this.fileDirectoryPath = fileDirectoryPath; - this.id = null; - this.createdDate = new Date(); - } - - /** - * Get the implementation-specific file label. - * - * @return the file label. - */ - @Override - public String getLabel() { - return filePath.getFileName().toString(); - } - - @Override - public Date getCreatedDate() { - return this.createdDate; - } - - @Override - public Long getId() { - return this.id; - } - - public Path getFilePath() { - return filePath; - } - - public Path getFileDirectoryPath() { - return fileDirectoryPath; - } - -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java index 5f3b8398e11..5d88f46cb2d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyHistoriesService.java @@ -29,8 +29,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.github.jmchilton.blend4j.galaxy.ToolsClient; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java deleted file mode 100644 index a5a1a0e8f46..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/submission/AnalysisSubmissionTempFileRepository.java +++ /dev/null @@ -1,25 +0,0 @@ -package ca.corefacility.bioinformatics.irida.repositories.analysis.submission; - -import java.util.List; - -import org.springframework.data.jpa.repository.Query; - -import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; -import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; -import ca.corefacility.bioinformatics.irida.repositories.IridaJpaRepository; - -/** - * A repository for managing {@link AnalysisSubmissionTempFile} objects. - */ - -public interface AnalysisSubmissionTempFileRepository extends IridaJpaRepository { - - /** - * Get all {@link AnalysisSubmissionTempFile} objects by submission id. - * - * @param analysisSubmission The {@link AnalysisSubmission} - * @return a list of {@link AnalysisSubmissionTempFile} - */ - @Query("FROM AnalysisSubmissionTempFile f WHERE f.analysisSubmission = ?1") - List findAllByAnalysisSubmission(AnalysisSubmission analysisSubmission); -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index 9a71167451a..b113502adaa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -7,13 +7,10 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.DatasetCollectionType; -import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; import ca.corefacility.bioinformatics.irida.pipeline.upload.DataStorage; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyHistoriesService; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; -import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import com.github.jmchilton.blend4j.galaxy.beans.History; import com.github.jmchilton.blend4j.galaxy.beans.Library; @@ -44,7 +41,6 @@ public class AnalysisCollectionServiceGalaxy { private GalaxyHistoriesService galaxyHistoriesService; private IridaFileStorageUtility iridaFileStorageUtility; - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; private DataStorage dataStorageType; /** @@ -54,14 +50,11 @@ public class AnalysisCollectionServiceGalaxy { * @param galaxyHistoriesService A GalaxyHistoriesService for interacting with * Galaxy Histories. * @param iridaFileStorageUtility The file storage utility implementation - * @param analysisSubmissionTempFileRepository {@link AnalysisSubmissionTempFileRepository} for {@link AnalysisSubmissionTempFile} objects */ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesService, - IridaFileStorageUtility iridaFileStorageUtility, - AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { + IridaFileStorageUtility iridaFileStorageUtility) { this.galaxyHistoriesService = galaxyHistoriesService; this.iridaFileStorageUtility = iridaFileStorageUtility; - this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; this.dataStorageType = iridaFileStorageUtility.isStorageTypeLocal() ? DataStorage.LOCAL : DataStorage.REMOTE; } @@ -73,7 +66,6 @@ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesSer * {@link SingleEndSequenceFile}. * @param workflowHistory The history to upload the sequence files into. * @param workflowLibrary A temporary library to upload files into. - * @param analysisSubmission The {@link AnalysisSubmission} * @return A CollectionResponse for the dataset collection constructed from the * given files. * @throws ExecutionManagerException If there was an error uploading the files. @@ -81,28 +73,18 @@ public AnalysisCollectionServiceGalaxy(GalaxyHistoriesService galaxyHistoriesSer */ public CollectionResponse uploadSequenceFilesSingleEnd( Map sampleSequenceFiles, History workflowHistory, - Library workflowLibrary, AnalysisSubmission analysisSubmission) throws ExecutionManagerException, IOException { + Library workflowLibrary) throws ExecutionManagerException, IOException { CollectionDescription description = new CollectionDescription(); description.setCollectionType(DatasetCollectionType.LIST.toString()); description.setName(COLLECTION_NAME_SINGLE); + IridaTemporaryFile iridaTemporaryFile = null; Map samplesMap = new HashMap<>(); for (Sample sample : sampleSequenceFiles.keySet()) { SingleEndSequenceFile sequenceFile = sampleSequenceFiles.get(sample); - IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile(), "analysis"); + iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile(), "analysis"); samplesMap.put(iridaTemporaryFile.getFile(), sample); - - /* - If the storage type is an object store then we create an AnalysisSubmissionTempFile - record to save to the database. The cleanup of these files happens once the final - workflow status is set for an analysis. - */ - if (!iridaFileStorageUtility.isStorageTypeLocal()) { - AnalysisSubmissionTempFile analysisSubmissionTempFile = new AnalysisSubmissionTempFile(analysisSubmission, - iridaTemporaryFile.getFile(), iridaTemporaryFile.getDirectoryPath()); - analysisSubmissionTempFileRepository.save(analysisSubmissionTempFile); - } } // upload files to library and then to a history @@ -123,7 +105,9 @@ record to save to the database. The cleanup of these files happens once the fina description.addDatasetElement(datasetElement); } - + if(iridaTemporaryFile != null) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + } return galaxyHistoriesService.constructCollection(description, workflowHistory); } @@ -136,7 +120,6 @@ record to save to the database. The cleanup of these files happens once the fina * @param workflowHistory The history to upload the sequence files * into. * @param workflowLibrary A temporary library to upload files into. - * @param analysisSubmission The {@link AnalysisSubmission} * @return A CollectionResponse for the dataset collection constructed from the * given files. * @throws ExecutionManagerException If there was an error uploading the files. @@ -144,12 +127,15 @@ record to save to the database. The cleanup of these files happens once the fina */ public CollectionResponse uploadSequenceFilesPaired( Map sampleSequenceFilesPaired, History workflowHistory, - Library workflowLibrary, AnalysisSubmission analysisSubmission) throws ExecutionManagerException, IOException { + Library workflowLibrary) throws ExecutionManagerException, IOException { CollectionDescription description = new CollectionDescription(); description.setCollectionType(DatasetCollectionType.LIST_PAIRED.toString()); description.setName(COLLECTION_NAME_PAIRED); + IridaTemporaryFile iridaTemporaryFileForward = null; + IridaTemporaryFile iridaTemporaryFileReverse = null; + Map samplesMapPairForward = new HashMap<>(); Map samplesMapPairReverse = new HashMap<>(); Set pathsToUpload = new HashSet<>(); @@ -158,29 +144,13 @@ public CollectionResponse uploadSequenceFilesPaired( SequenceFile fileForward = sequenceFilePair.getForwardSequenceFile(); SequenceFile fileReverse = sequenceFilePair.getReverseSequenceFile(); - IridaTemporaryFile iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile(), "analysis"); - IridaTemporaryFile iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile(), "analysis"); + iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile(), "analysis"); + iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile(), "analysis"); samplesMapPairForward.put(sample, iridaTemporaryFileForward.getFile()); samplesMapPairReverse.put(sample, iridaTemporaryFileReverse.getFile()); pathsToUpload.add(iridaTemporaryFileForward.getFile()); pathsToUpload.add(iridaTemporaryFileReverse.getFile()); - - /* - If the storage type is an object store then we create an AnalysisSubmissionTempFile - record to save to the database. The cleanup of these files happens once the final - workflow status is set for an analysis. - */ - if (!iridaFileStorageUtility.isStorageTypeLocal()) { - AnalysisSubmissionTempFile analysisSubmissionTempFileForward = new AnalysisSubmissionTempFile( - analysisSubmission, iridaTemporaryFileForward.getFile(), - iridaTemporaryFileForward.getDirectoryPath()); - AnalysisSubmissionTempFile analysisSubmissionTempFileReverse = new AnalysisSubmissionTempFile( - analysisSubmission, iridaTemporaryFileReverse.getFile(), - iridaTemporaryFileReverse.getDirectoryPath()); - analysisSubmissionTempFileRepository.save(analysisSubmissionTempFileForward); - analysisSubmissionTempFileRepository.save(analysisSubmissionTempFileReverse); - } } // upload files to library and then to a history @@ -216,6 +186,13 @@ record to save to the database. The cleanup of these files happens once the fina description.addDatasetElement(pairedElement); } } + if(iridaTemporaryFileForward != null) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFileForward); + } + + if(iridaTemporaryFileReverse != null) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFileReverse); + } return galaxyHistoriesService.constructCollection(description, workflowHistory); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java index 2d5ef548e5f..3ea0af23d19 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java @@ -226,7 +226,7 @@ public PreparedWorkflowGalaxy prepareAnalysisFiles(AnalysisSubmission analysisSu String workflowSequenceFileSingleInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, sequenceFilesLabelSingle); CollectionResponse collectionResponseSingle = analysisCollectionServiceGalaxy - .uploadSequenceFilesSingleEnd(singleFiles, workflowHistory, workflowLibrary, analysisSubmission); + .uploadSequenceFilesSingleEnd(singleFiles, workflowHistory, workflowLibrary); inputs.setInput(workflowSequenceFileSingleInputId, new WorkflowInvocationInputs.WorkflowInvocationInput( collectionResponseSingle.getId(), WorkflowInvocationInputs.InputSourceType.HDCA)); } @@ -236,7 +236,7 @@ public PreparedWorkflowGalaxy prepareAnalysisFiles(AnalysisSubmission analysisSu String workflowSequenceFilePairedInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, sequenceFilesLabelPaired); CollectionResponse collectionResponsePaired = analysisCollectionServiceGalaxy - .uploadSequenceFilesPaired(pairedFiles, workflowHistory, workflowLibrary, analysisSubmission); + .uploadSequenceFilesPaired(pairedFiles, workflowHistory, workflowLibrary); inputs.setInput(workflowSequenceFilePairedInputId, new WorkflowInvocationInputs.WorkflowInvocationInput( collectionResponsePaired.getId(), WorkflowInvocationInputs.InputSourceType.HDCA)); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java index b231511beff..e18b7c2b2ee 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/AnalysisExecutionScheduledTaskImpl.java @@ -19,20 +19,15 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.JobError; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; -import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmissionTempFile; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; import ca.corefacility.bioinformatics.irida.service.EmailController; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; -import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import com.google.common.collect.Sets; @@ -58,9 +53,7 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche private GalaxyJobErrorsService galaxyJobErrorsService; private JobErrorRepository jobErrorRepository; private EmailController emailController; - private IridaFileStorageUtility iridaFileStorageUtility; private AnalysisWorkspaceService analysisWorkspaceService; - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; /** * Builds a new AnalysisExecutionScheduledTaskImpl with the given service @@ -74,24 +67,19 @@ public class AnalysisExecutionScheduledTaskImpl implements AnalysisExecutionSche * @param jobErrorRepository {@link JobErrorRepository} for {@link JobError} objects * @param emailController {@link EmailController} for sending completion/error emails for {@link AnalysisSubmission}s * @param analysisWorkspaceService {@link AnalysisWorkspaceService} - * @param iridaFileStorageUtility The irida file storage utility implementation - * @param analysisSubmissionTempFileRepository {@link AnalysisSubmissionTempFileRepository} for {@link AnalysisSubmissionTempFile} objects */ @Autowired public AnalysisExecutionScheduledTaskImpl(AnalysisSubmissionRepository analysisSubmissionRepository, AnalysisExecutionService analysisExecutionServiceGalaxy, CleanupAnalysisSubmissionCondition cleanupCondition, GalaxyJobErrorsService galaxyJobErrorsService, - JobErrorRepository jobErrorRepository, EmailController emailController, AnalysisWorkspaceService analysisWorkspaceService, IridaFileStorageUtility iridaFileStorageUtility, - AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository) { + JobErrorRepository jobErrorRepository, EmailController emailController, AnalysisWorkspaceService analysisWorkspaceService) { this.analysisSubmissionRepository = analysisSubmissionRepository; this.analysisExecutionService = analysisExecutionServiceGalaxy; this.cleanupCondition = cleanupCondition; this.galaxyJobErrorsService = galaxyJobErrorsService; this.jobErrorRepository = jobErrorRepository; this.emailController = emailController; - this.iridaFileStorageUtility = iridaFileStorageUtility; this.analysisWorkspaceService = analysisWorkspaceService; - this.analysisSubmissionTempFileRepository = analysisSubmissionTempFileRepository; } /** @@ -193,9 +181,6 @@ public Set> monitorRunningAnalyses() { analysisSubmission.setAnalysisState(AnalysisState.ERROR); submissions.add(new AsyncResult<>(analysisSubmissionRepository.save(analysisSubmission))); - // Clean up any downloaded temp files - cleanupTemporaryDownloadedFiles(analysisSubmission); - if (analysisSubmission.getEmailPipelineResultError()) { emailController.sendPipelineStatusEmail(analysisSubmission); } @@ -322,9 +307,6 @@ private Future handleWorkflowStatus(GalaxyWorkflowStatus wor */ if (finalWorkflowStatusSet) { - // Clean up any downloaded temp files - cleanupTemporaryDownloadedFiles(analysisSubmission); - /* If the analysis has finished with an error or completed successfully and the user selected to be emailed on completion or error, then the following code @@ -373,30 +355,4 @@ public Set> cleanupAnalysisSubmissions() { return cleanedSubmissions; } } - - /** - * Cleanup any temporary downloaded files and cleanup associated {@link AnalysisSubmissionTempFile} objects - * - * @param submission The analysis submission to clean up temporary downloaded files for - */ - private void cleanupTemporaryDownloadedFiles(AnalysisSubmission submission) { - /* - Cleanup any files that were downloaded from an object store to run an analysis and - remove the analysis submission temp file record from the database. - */ - List analysisSubmissionTempFiles = analysisSubmissionTempFileRepository.findAllByAnalysisSubmission( - submission); - - if (analysisSubmissionTempFiles.size() > 0) { - logger.debug("Cleaning up " + analysisSubmissionTempFiles.size() - + " temporary files downloaded from object store."); - } - - for (AnalysisSubmissionTempFile analysisSubmissionTempFile : analysisSubmissionTempFiles) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles( - new IridaTemporaryFile(analysisSubmissionTempFile.getFilePath(), - analysisSubmissionTempFile.getFileDirectoryPath())); - analysisSubmissionTempFileRepository.delete(analysisSubmissionTempFile); - } - } } diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml index 97dbfc31659..91ad35bc0df 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/all-changes.xml @@ -12,7 +12,5 @@ relativeToChangelogFile="true"/> - diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml deleted file mode 100644 index 6f79646045c..00000000000 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/21.01/analysis-submission-cloud-temp-files.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java index 0c7319f5015..fbd7961958d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/config/analysis/AnalysisExecutionServiceTestConfig.java @@ -33,7 +33,6 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyWorkflowService; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.referencefile.ReferenceFileRepository; import ca.corefacility.bioinformatics.irida.repositories.sample.SampleRepository; @@ -105,9 +104,6 @@ public class AnalysisExecutionServiceTestConfig { @Autowired private SampleRepository sampleRepository; - @Autowired - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Autowired private IridaFileStorageUtility iridaFileStorageUtility; @@ -152,8 +148,7 @@ galaxyLibrariesService, iridaWorkflowsService, analysisCollectionServiceGalaxy() @Lazy @Bean public AnalysisCollectionServiceGalaxy analysisCollectionServiceGalaxy() { - return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository); + return new AnalysisCollectionServiceGalaxy(galaxyHistoriesService, iridaFileStorageUtility); } @Lazy diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java index 1f742ae4de1..d48fe7a0fa6 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/enums/analysis/integration/SNVPhylAnalysisIT.java @@ -58,9 +58,7 @@ import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; @@ -131,13 +129,6 @@ public class SNVPhylAnalysisIT { private AnalysisExecutionScheduledTask analysisExecutionScheduledTask; - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - - @Autowired - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - - private Path sequenceFilePathA1; private Path sequenceFilePathA2; private Path sequenceFilePathB1; @@ -183,8 +174,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.NEVER_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService); Path tempDir = Files.createTempDirectory(rootTempDirectory, "snvphylTest"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java index d9b4b720f56..252b491b984 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyJobErrorsServiceIT.java @@ -29,9 +29,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.EmailController; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; @@ -85,12 +83,6 @@ public class GalaxyJobErrorsServiceIT { @Autowired private AnalysisWorkspaceService analysisWorkspaceService; - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - - @Autowired - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - @Before public void setup() throws URISyntaxException, IOException { @@ -110,8 +102,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java index c92b1feca8c..7f24d61cfb8 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java @@ -16,7 +16,6 @@ import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.DatasetCollectionType; -import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; import ca.corefacility.bioinformatics.irida.processing.impl.GzipFileProcessor; import ca.corefacility.bioinformatics.irida.repositories.referencefile.ReferenceFileRepository; @@ -137,7 +136,6 @@ public class AnalysisCollectionServiceGalaxyIT { private static final String FORWARD_NAME = "forward"; private static final String REVERSE_NAME = "reverse"; - private AnalysisSubmission analysisSubmission; private SingleEndSequenceFile singleEndFile; private SequenceFile sequenceFile; private UUID workflowId = UUID.randomUUID(); @@ -218,13 +216,6 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc sequenceFile = singleEndFile.getFileWithId(1L); assertNotNull(sequenceFile); Set singleFiles = Sets.newHashSet(singleEndFile); - - analysisSubmission = AnalysisSubmission.builder(workflowId).name(analysisName).inputFiles(singleFiles) - .referenceFile(referenceFile).build(); - analysisSubmission.setRemoteAnalysisId(analysisId); - analysisSubmission.setAnalysisState(AnalysisState.SUBMITTING); - analysisSubmission.setSubmitter(submitter1); - analysisSubmission.setAnalysisCleanedState(AnalysisCleanedState.NOT_CLEANED); } /** @@ -327,7 +318,7 @@ public void testUploadSequenceFilesSingleSuccess() throws ExecutionManagerExcept Sample sample1 = sampleRepository.findById(1L).orElse(null); CollectionResponse collectionResponse = analysisCollectionServiceGalaxy - .uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, createdLibrary, analysisSubmission); + .uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, createdLibrary); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -391,7 +382,7 @@ public void testUploadSequenceFilesSingleCompressedSuccess() throws ExecutionMan sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(sampleSequenceFiles, createdHistory, - createdLibrary, analysisSubmission); + createdLibrary); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -439,7 +430,7 @@ public void testUploadSequenceFilesPairedSuccess() throws ExecutionManagerExcept Sample sample1 = sampleRepository.findById(1L).orElse(null); CollectionResponse collectionResponse = analysisCollectionServiceGalaxy - .uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, createdLibrary, analysisSubmission); + .uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, createdLibrary); // verify correct files have been uploaded List historyContents = historiesClient.showHistoryContents(createdHistory.getId()); @@ -538,7 +529,7 @@ public void testUploadSequenceFilesPairedCompressedSuccess() throws ExecutionMan sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, - createdLibrary, analysisSubmission); + createdLibrary); SequenceFilePair pairedSequenceFile = sequenceFiles.iterator().next(); for (SequenceFile file : pairedSequenceFile.getFiles()) { @@ -594,7 +585,7 @@ public void testUploadSequenceFilesPairedFailForward() throws ExecutionManagerEx Map sampleSequenceFilePairs = new HashMap<>(sequencingObjectService.getUniqueSamplesForSequencingObjects(sequenceFiles)); analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(sampleSequenceFilePairs, createdHistory, - createdLibrary, analysisSubmission); + createdLibrary); } private Map historyContentsAsMap(List historyContents) { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java index 99502401efd..caf89c6d6dc 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/unit/AnalysisWorkspaceServiceGalaxyTest.java @@ -304,9 +304,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponseSingle); + eq(workflowLibrary))).thenReturn(collectionResponseSingle); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponsePaired); + eq(workflowLibrary))).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -324,9 +324,9 @@ public void testPrepareAnalysisFilesSinglePairedSuccess() throws ExecutionManage assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class), any(AnalysisSubmission.class)); + any(Library.class)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), any(AnalysisSubmission.class)); + any(Library.class)); } /** @@ -369,7 +369,7 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesSingleEnd(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponseSingle); + eq(workflowLibrary))).thenReturn(collectionResponseSingle); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -386,9 +386,9 @@ public void testPrepareAnalysisFilesSingleSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file single entry", workflowInputsMap.containsKey(SEQUENCE_FILE_SINGLE_ID)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesSingleEnd(any(Map.class), any(History.class), - any(Library.class), any(AnalysisSubmission.class)); + any(Library.class)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), any(AnalysisSubmission.class)); + any(Library.class)); } /** @@ -432,7 +432,7 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep .thenReturn(REFERENCE_FILE_ID); when(analysisCollectionServiceGalaxy.uploadSequenceFilesPaired(any(Map.class), eq(workflowHistory), - eq(workflowLibrary), any(AnalysisSubmission.class))).thenReturn(collectionResponsePaired); + eq(workflowLibrary))).thenReturn(collectionResponsePaired); PreparedWorkflowGalaxy preparedWorkflow = workflowPreparation.prepareAnalysisFiles(submission); @@ -449,9 +449,9 @@ public void testPrepareAnalysisFilesPairedSuccess() throws ExecutionManagerExcep assertTrue("workflow inputs should contain sequence file paired entry", workflowInputsMap.containsKey(SEQUENCE_FILE_PAIRED_ID)); verify(analysisCollectionServiceGalaxy, never()).uploadSequenceFilesSingleEnd(any(Map.class), - any(History.class), any(Library.class), any(AnalysisSubmission.class)); + any(History.class), any(Library.class)); verify(analysisCollectionServiceGalaxy).uploadSequenceFilesPaired(any(Map.class), any(History.class), - any(Library.class), any(AnalysisSubmission.class)); + any(Library.class)); } /** diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java index 20ce86a1e33..ed0bdd3ef04 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/integration/AnalysisExecutionScheduledTaskImplIT.java @@ -40,9 +40,7 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyJobErrorsService; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionRepository; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.JobErrorRepository; -import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.repositories.user.UserRepository; import ca.corefacility.bioinformatics.irida.service.EmailController; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; @@ -99,12 +97,6 @@ public class AnalysisExecutionScheduledTaskImplIT { private AnalysisExecutionScheduledTask analysisExecutionScheduledTask; - @Autowired - private IridaFileStorageUtility iridaFileStorageUtility; - - @Autowired - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - private Path sequenceFilePath; private Path sequenceFilePath2; @@ -129,8 +121,7 @@ public void setup() throws URISyntaxException, IOException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService); Path sequenceFilePathReal = Paths .get(DatabaseSetupGalaxyITService.class.getResource("testData1.fastq").toURI()); @@ -197,8 +188,7 @@ public void testFullAnalysisRunSuccessWithSampleUpdates() throws Exception { public void testFullAnalysisRunSuccessNoCleanupAge() throws Exception { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService); AnalysisSubmission analysisSubmission = analysisExecutionGalaxyITService.setupSubmissionInDatabase(1L, sequenceFilePath, referenceFilePath, validIridaWorkflowId, false); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java index 0e472292668..7317b6d472f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/impl/unit/AnalysisExecutionScheduledTaskImplTest.java @@ -12,7 +12,6 @@ import java.util.UUID; import java.util.concurrent.Future; -import ca.corefacility.bioinformatics.irida.repositories.analysis.submission.AnalysisSubmissionTempFileRepository; import ca.corefacility.bioinformatics.irida.service.analysis.workspace.AnalysisWorkspaceService; import org.joda.time.DateTime; import org.junit.Before; @@ -20,7 +19,6 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import com.github.jmchilton.blend4j.galaxy.HistoriesClient; import com.google.common.collect.Sets; import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; @@ -31,7 +29,6 @@ import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.model.workflow.analysis.Analysis; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowState; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.GalaxyWorkflowStatus; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; @@ -42,7 +39,6 @@ import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.service.AnalysisExecutionScheduledTask; -import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.CleanupAnalysisSubmissionCondition; import ca.corefacility.bioinformatics.irida.service.analysis.execution.AnalysisExecutionService; import ca.corefacility.bioinformatics.irida.service.impl.AnalysisExecutionScheduledTaskImpl; @@ -54,8 +50,6 @@ */ public class AnalysisExecutionScheduledTaskImplTest { - @Mock - private AnalysisSubmissionService analysisSubmissionService; @Mock private AnalysisSubmissionRepository analysisSubmissionRepository; @Mock @@ -67,9 +61,6 @@ public class AnalysisExecutionScheduledTaskImplTest { @Mock private ReferenceFile referenceFile; - @Mock - private Analysis analysis; - @Mock private AnalysisSubmission analysisSubmissionMock; @@ -79,9 +70,6 @@ public class AnalysisExecutionScheduledTaskImplTest { @Mock private GalaxyJobErrorsService galaxyJobErrorsService; - @Mock - private HistoriesClient historiesClient; - @Mock private JobErrorRepository jobErrorRepository; @@ -91,9 +79,6 @@ public class AnalysisExecutionScheduledTaskImplTest { @Mock private AnalysisWorkspaceService analysisWorkspaceService; - @Mock - private AnalysisSubmissionTempFileRepository analysisSubmissionTempFileRepository; - private static final String ANALYSIS_ID = "1"; private static final Long INTERNAL_ID = 1L; private AnalysisSubmission analysisSubmission; @@ -114,8 +99,7 @@ public void setup() { iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, CleanupAnalysisSubmissionCondition.ALWAYS_CLEANUP, galaxyJobErrorsService, - jobErrorRepository, emailController, analysisWorkspaceService, iridaFileStorageUtility, - analysisSubmissionTempFileRepository); + jobErrorRepository, emailController, analysisWorkspaceService); analysisSubmission = AnalysisSubmission.builder(workflowId) .name("my analysis") @@ -680,8 +664,7 @@ public void testCleanupAnalysisSubmissionsCompletedCleanedCleaningSuccess() thro public void testCleanupAnalysisSubmissionsCompletedOverOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -708,8 +691,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedCleanupZeroSuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZERO), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -736,8 +718,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ZER public void testCleanupAnalysisSubmissionsCompletedUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); @@ -764,8 +745,7 @@ analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofD public void testCleanupAnalysisSubmissionsCompletedOverUnderOneDaySuccess() throws ExecutionManagerException { analysisExecutionScheduledTask = new AnalysisExecutionScheduledTaskImpl(analysisSubmissionRepository, analysisExecutionService, new CleanupAnalysisSubmissionConditionAge(Duration.ofDays(1)), - galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService, - iridaFileStorageUtility, analysisSubmissionTempFileRepository); + galaxyJobErrorsService, jobErrorRepository, emailController, analysisWorkspaceService); when(analysisSubmissionMock.getAnalysisState()).thenReturn(AnalysisState.COMPLETED); when(analysisSubmissionMock.getAnalysisCleanedState()).thenReturn(AnalysisCleanedState.NOT_CLEANED); From 50d4f4fd7d2a884fc1dc8abc61771bc106536aad Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 12 Jul 2021 14:38:59 -0500 Subject: [PATCH 214/655] Removed unused variables, fixed formatting, removed StorageType logic in file storage utility classes when checking if storage type is local or not and replaced with true/false --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 5 +---- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 5 +---- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 6 ++---- .../bioinformatics/irida/util/IridaFiles.java | 8 +++++--- .../galaxy/integration/GalaxyHistoriesServiceIT.java | 1 - .../integration/AnalysisCollectionServiceGalaxyIT.java | 10 ---------- 6 files changed, 9 insertions(+), 26 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 1bb951a2ab8..a2045d95d7a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -16,7 +16,6 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -41,7 +40,6 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private String bucketName; private BasicAWSCredentials awsCreds; private AmazonS3 s3; - private StorageType storageType; @Autowired public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { @@ -51,7 +49,6 @@ public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, St .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .build(); this.bucketName = bucketName; - this.storageType = StorageType.AWS; } /** @@ -392,6 +389,6 @@ public boolean checkWriteAccess(Path baseDirectory) { */ @Override public boolean isStorageTypeLocal() { - return storageType.equals(StorageType.LOCAL); + return false; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 57774b2e981..75f5f480655 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -39,7 +38,6 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; - private StorageType storageType; @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { @@ -47,7 +45,6 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St .sasToken(sasToken) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - this.storageType = StorageType.AZURE; } /** @@ -392,6 +389,6 @@ public boolean checkWriteAccess(Path baseDirectory) { */ @Override public boolean isStorageTypeLocal() { - return storageType.equals(StorageType.LOCAL); + return false; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index f626623a8ae..78742ce527d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -18,7 +18,6 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; -import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -32,11 +31,10 @@ public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); - private StorageType storageType; @Autowired public IridaFileStorageLocalUtilityImpl() { - this.storageType = StorageType.LOCAL; + } /** @@ -290,6 +288,6 @@ public boolean checkWriteAccess(Path baseDirectory) { */ @Override public boolean isStorageTypeLocal() { - return storageType.equals(StorageType.LOCAL); + return true; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java index 3074eeb7d67..69d668d1ee2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/util/IridaFiles.java @@ -35,7 +35,7 @@ public static String getFileSize(Path file) { String fileSize = "N/A"; Long fileSizeBytes = iridaFileStorageUtility.getFileSizeBytes(file); - if(fileSizeBytes > 0){ + if (fileSizeBytes > 0) { fileSize = FileUtils.humanReadableByteCount(fileSizeBytes, true); } return fileSize; @@ -80,7 +80,7 @@ public static String getFileExtension(List files) th * @return the bytes for the file * @throws IOException if the file couldn't be read */ - public static byte[] getBytesForFile(Path file) throws IOException { + public static byte[] getBytesForFile(Path file) throws IOException { byte[] bytes = iridaFileStorageUtility.readAllBytes(file); return bytes; } @@ -101,6 +101,8 @@ public static Long getFileSizeBytes(Path file) { * @param file The path to the file * @return if file exists or not */ - public static boolean fileExists(Path file) { return iridaFileStorageUtility.fileExists(file);} + public static boolean fileExists(Path file) { + return iridaFileStorageUtility.fileExists(file); + } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java index 08320f9c7c7..561e7623f0a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/integration/GalaxyHistoriesServiceIT.java @@ -103,7 +103,6 @@ public class GalaxyHistoriesServiceIT { */ private static final int LIBRARY_POLLING_TIME = 5; - private static final Long ANALYSIS_SUBMISSION_ID = 1L; /** * Sets up files for history tests. diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java index 7f24d61cfb8..481615638a4 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/impl/integration/AnalysisCollectionServiceGalaxyIT.java @@ -5,15 +5,12 @@ import ca.corefacility.bioinformatics.irida.exceptions.DuplicateSampleException; import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; import ca.corefacility.bioinformatics.irida.exceptions.IridaWorkflowLoadException; -import ca.corefacility.bioinformatics.irida.model.enums.AnalysisCleanedState; -import ca.corefacility.bioinformatics.irida.model.enums.AnalysisState; import ca.corefacility.bioinformatics.irida.model.project.ReferenceFile; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SingleEndSequenceFile; -import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.execution.InputFileType; import ca.corefacility.bioinformatics.irida.model.workflow.execution.galaxy.DatasetCollectionType; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.integration.LocalGalaxy; @@ -141,11 +138,6 @@ public class AnalysisCollectionServiceGalaxyIT { private UUID workflowId = UUID.randomUUID(); private ReferenceFile referenceFile; - private static final String analysisId = "1"; - - private final String analysisName = "analysis 1"; - - private User submitter1; /** * Sets up variables for testing. @@ -209,8 +201,6 @@ public void setup() throws URISyntaxException, IOException, IridaWorkflowLoadExc gzipFileProcessor.setDisableFileProcessor(false); - submitter1 = userRepository.findById(1L).orElse(null); - referenceFile = referenceFileRepository.findById(1L).orElse(null); singleEndFile = (SingleEndSequenceFile) objectRepository.findById(2L).orElse(null); sequenceFile = singleEndFile.getFileWithId(1L); From 55799501f82e13585f8b7a670ee3feace53f349c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 14 Jul 2021 14:00:19 -0500 Subject: [PATCH 215/655] Updated logic to cleanup a list of iridatemporaryfiles. Added try/catch block to cleaning up temporary downloaded files so if an exception is thrown we try to remove the temp file again so that it is not left behind --- .../AnalysisCollectionServiceGalaxy.java | 69 +++++++++++++++---- .../AnalysisWorkspaceServiceGalaxy.java | 13 +++- 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index b113502adaa..df9730d0786 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.service.analysis.workspace.galaxy; import ca.corefacility.bioinformatics.irida.exceptions.ExecutionManagerException; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.exceptions.UploadException; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -20,11 +21,9 @@ import com.github.jmchilton.blend4j.galaxy.beans.collection.response.CollectionResponse; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * A service for constructing dataset collections of input files for workflows @@ -79,11 +78,14 @@ public CollectionResponse uploadSequenceFilesSingleEnd( description.setCollectionType(DatasetCollectionType.LIST.toString()); description.setName(COLLECTION_NAME_SINGLE); - IridaTemporaryFile iridaTemporaryFile = null; + IridaTemporaryFile iridaTemporaryFile; + List filesToCleanUp = new ArrayList<>(); + Map samplesMap = new HashMap<>(); for (Sample sample : sampleSequenceFiles.keySet()) { SingleEndSequenceFile sequenceFile = sampleSequenceFiles.get(sample); iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile(), "analysis"); + filesToCleanUp.add(iridaTemporaryFile); samplesMap.put(iridaTemporaryFile.getFile(), sample); } @@ -105,8 +107,17 @@ public CollectionResponse uploadSequenceFilesSingleEnd( description.addDatasetElement(datasetElement); } - if(iridaTemporaryFile != null) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + + if(filesToCleanUp.size() > 0) { + for(IridaTemporaryFile itf : filesToCleanUp) { + try { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(itf); + } catch (StorageException e) { + throw new StorageException(e.getMessage()); + } finally { + Files.delete(itf.getFile()); + } + } } return galaxyHistoriesService.constructCollection(description, workflowHistory); } @@ -133,8 +144,10 @@ public CollectionResponse uploadSequenceFilesPaired( description.setCollectionType(DatasetCollectionType.LIST_PAIRED.toString()); description.setName(COLLECTION_NAME_PAIRED); - IridaTemporaryFile iridaTemporaryFileForward = null; - IridaTemporaryFile iridaTemporaryFileReverse = null; + IridaTemporaryFile iridaTemporaryFileForward; + IridaTemporaryFile iridaTemporaryFileReverse; + + List filesToCleanUp = new ArrayList<>(); Map samplesMapPairForward = new HashMap<>(); Map samplesMapPairReverse = new HashMap<>(); @@ -147,6 +160,9 @@ public CollectionResponse uploadSequenceFilesPaired( iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile(), "analysis"); iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile(), "analysis"); + filesToCleanUp.add(iridaTemporaryFileForward); + filesToCleanUp.add(iridaTemporaryFileReverse); + samplesMapPairForward.put(sample, iridaTemporaryFileForward.getFile()); samplesMapPairReverse.put(sample, iridaTemporaryFileReverse.getFile()); pathsToUpload.add(iridaTemporaryFileForward.getFile()); @@ -186,14 +202,39 @@ public CollectionResponse uploadSequenceFilesPaired( description.addDatasetElement(pairedElement); } } - if(iridaTemporaryFileForward != null) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFileForward); - } - if(iridaTemporaryFileReverse != null) { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFileReverse); + if(filesToCleanUp.size() > 0) { + try { + cleanupTemporaryGalaxyFiles(filesToCleanUp); + } catch (StorageException e) { + throw new StorageException(e.getMessage()); + } catch (IOException e) { + throw new IOException(e.getMessage()); + } } return galaxyHistoriesService.constructCollection(description, workflowHistory); } + + /** + * Clean up any temporary files that were downloaded and uploaded to Galaxy + * from an object store + * + * @param filesToCleanUp A list of {@link IridaTemporaryFile}'s to cleanup. + * @throws IOException If there was an error cleaning up the temporary file + */ + private void cleanupTemporaryGalaxyFiles(List filesToCleanUp) throws IOException { + for (IridaTemporaryFile itf : filesToCleanUp) { + /* + * If there was an error when cleaning up the downloaded temporary files then throw an exception + * and try to delete the temporary file again so it is not left behind + */ + try { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(itf); + } catch (StorageException e) { + Files.delete(itf.getFile()); + throw new StorageException(e.getMessage()); + } + } + } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java index 3ea0af23d19..c5f458122b9 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java @@ -299,7 +299,7 @@ private boolean samplesInCommon(Map sampleSequenceFilesSingle, */ private void prepareReferenceFile(ReferenceFile referenceFile, History workflowHistory, String referenceFileLabel, WorkflowDetails workflowDetails, WorkflowInvocationInputs inputs) - throws UploadException, GalaxyDatasetException, WorkflowException { + throws UploadException, GalaxyDatasetException, WorkflowException, IOException { IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(referenceFile.getFile()); @@ -312,7 +312,16 @@ private void prepareReferenceFile(ReferenceFile referenceFile, History workflowH inputs.setInput(workflowReferenceFileInputId, new WorkflowInvocationInputs.WorkflowInvocationInput(referenceDataset.getId(), WorkflowInvocationInputs.InputSourceType.HDA)); - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + /* + * If there was an error when cleaning up the downloaded temporary files then throw an exception + * and try to delete the temporary file again so it is not left behind + */ + try { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + } catch (StorageException e) { + Files.delete(iridaTemporaryFile.getFile()); + throw new StorageException(e.getMessage()); + } } /** From ed329514516fc78562b1070af2fde122f529ae42 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 15 Jul 2021 14:57:16 -0500 Subject: [PATCH 216/655] Moved logic into try/finally block so temp files get cleared even if an exception is thrown --- .../AnalysisCollectionServiceGalaxy.java | 209 +++++++++--------- .../AnalysisWorkspaceServiceGalaxy.java | 30 ++- 2 files changed, 113 insertions(+), 126 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index df9730d0786..ed168878e87 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -74,52 +74,48 @@ public CollectionResponse uploadSequenceFilesSingleEnd( Map sampleSequenceFiles, History workflowHistory, Library workflowLibrary) throws ExecutionManagerException, IOException { - CollectionDescription description = new CollectionDescription(); - description.setCollectionType(DatasetCollectionType.LIST.toString()); - description.setName(COLLECTION_NAME_SINGLE); - - IridaTemporaryFile iridaTemporaryFile; List filesToCleanUp = new ArrayList<>(); - - Map samplesMap = new HashMap<>(); - for (Sample sample : sampleSequenceFiles.keySet()) { - SingleEndSequenceFile sequenceFile = sampleSequenceFiles.get(sample); - iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile().getFile(), "analysis"); - filesToCleanUp.add(iridaTemporaryFile); - samplesMap.put(iridaTemporaryFile.getFile(), sample); - } - - // upload files to library and then to a history - Map pathHistoryDatasetId = galaxyHistoriesService.filesToLibraryToHistory(samplesMap.keySet(), - workflowHistory, workflowLibrary, dataStorageType); - - for (Path sequenceFilePath : samplesMap.keySet()) { - if (!pathHistoryDatasetId.containsKey(sequenceFilePath)) { - throw new UploadException("Error, no corresponding history item found for " + sequenceFilePath); + try { + CollectionDescription description = new CollectionDescription(); + description.setCollectionType(DatasetCollectionType.LIST.toString()); + description.setName(COLLECTION_NAME_SINGLE); + + IridaTemporaryFile iridaTemporaryFile; + + Map samplesMap = new HashMap<>(); + for (Sample sample : sampleSequenceFiles.keySet()) { + SingleEndSequenceFile sequenceFile = sampleSequenceFiles.get(sample); + iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(sequenceFile.getSequenceFile() + .getFile(), "analysis"); + filesToCleanUp.add(iridaTemporaryFile); + samplesMap.put(iridaTemporaryFile.getFile(), sample); } - Sample sample = samplesMap.get(sequenceFilePath); - String datasetHistoryId = pathHistoryDatasetId.get(sequenceFilePath); + // upload files to library and then to a history + Map pathHistoryDatasetId = galaxyHistoriesService.filesToLibraryToHistory(samplesMap.keySet(), + workflowHistory, workflowLibrary, dataStorageType); - HistoryDatasetElement datasetElement = new HistoryDatasetElement(); - datasetElement.setId(datasetHistoryId); - datasetElement.setName(sample.getSampleName()); + for (Path sequenceFilePath : samplesMap.keySet()) { + if (!pathHistoryDatasetId.containsKey(sequenceFilePath)) { + throw new UploadException("Error, no corresponding history item found for " + sequenceFilePath); + } - description.addDatasetElement(datasetElement); - } + Sample sample = samplesMap.get(sequenceFilePath); + String datasetHistoryId = pathHistoryDatasetId.get(sequenceFilePath); - if(filesToCleanUp.size() > 0) { - for(IridaTemporaryFile itf : filesToCleanUp) { - try { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(itf); - } catch (StorageException e) { - throw new StorageException(e.getMessage()); - } finally { - Files.delete(itf.getFile()); - } + HistoryDatasetElement datasetElement = new HistoryDatasetElement(); + datasetElement.setId(datasetHistoryId); + datasetElement.setName(sample.getSampleName()); + + description.addDatasetElement(datasetElement); + } + return galaxyHistoriesService.constructCollection(description, workflowHistory); + } finally { + if (filesToCleanUp.size() > 0) { + cleanupTemporaryGalaxyFiles(filesToCleanUp); } } - return galaxyHistoriesService.constructCollection(description, workflowHistory); + } /** @@ -138,92 +134,86 @@ public CollectionResponse uploadSequenceFilesSingleEnd( */ public CollectionResponse uploadSequenceFilesPaired( Map sampleSequenceFilesPaired, History workflowHistory, - Library workflowLibrary) throws ExecutionManagerException, IOException { - - CollectionDescription description = new CollectionDescription(); - description.setCollectionType(DatasetCollectionType.LIST_PAIRED.toString()); - description.setName(COLLECTION_NAME_PAIRED); - - IridaTemporaryFile iridaTemporaryFileForward; - IridaTemporaryFile iridaTemporaryFileReverse; + Library workflowLibrary) throws ExecutionManagerException, IOException, StorageException { List filesToCleanUp = new ArrayList<>(); - - Map samplesMapPairForward = new HashMap<>(); - Map samplesMapPairReverse = new HashMap<>(); - Set pathsToUpload = new HashSet<>(); - for (Sample sample : sampleSequenceFilesPaired.keySet()) { - SequenceFilePair sequenceFilePair = sampleSequenceFilesPaired.get(sample); - SequenceFile fileForward = sequenceFilePair.getForwardSequenceFile(); - SequenceFile fileReverse = sequenceFilePair.getReverseSequenceFile(); - - iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile(), "analysis"); - iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile(), "analysis"); - - filesToCleanUp.add(iridaTemporaryFileForward); - filesToCleanUp.add(iridaTemporaryFileReverse); - - samplesMapPairForward.put(sample, iridaTemporaryFileForward.getFile()); - samplesMapPairReverse.put(sample, iridaTemporaryFileReverse.getFile()); - pathsToUpload.add(iridaTemporaryFileForward.getFile()); - pathsToUpload.add(iridaTemporaryFileReverse.getFile()); - } - - // upload files to library and then to a history - Map pathHistoryDatasetId = galaxyHistoriesService.filesToLibraryToHistory(pathsToUpload, - workflowHistory, workflowLibrary, dataStorageType); - - for (Sample sample : sampleSequenceFilesPaired.keySet()) { - Path fileForward = samplesMapPairForward.get(sample); - Path fileReverse = samplesMapPairReverse.get(sample); - - if (!pathHistoryDatasetId.containsKey(fileForward)) { - throw new UploadException("Error, no corresponding history item found for " + fileForward); - } else if (!pathHistoryDatasetId.containsKey(fileReverse)) { - throw new UploadException("Error, no corresponding history item found for " + fileReverse); - } else { - String datasetHistoryIdForward = pathHistoryDatasetId.get(fileForward); - String datasetHistoryIdReverse = pathHistoryDatasetId.get(fileReverse); - - CollectionElement pairedElement = new CollectionElement(); - pairedElement.setName(sample.getSampleName()); - pairedElement.setCollectionType(DatasetCollectionType.PAIRED.toString()); - - HistoryDatasetElement datasetElementForward = new HistoryDatasetElement(); - datasetElementForward.setId(datasetHistoryIdForward); - datasetElementForward.setName(FORWARD_NAME); - pairedElement.addCollectionElement(datasetElementForward); - - HistoryDatasetElement datasetElementReverse = new HistoryDatasetElement(); - datasetElementReverse.setId(datasetHistoryIdReverse); - datasetElementReverse.setName(REVERSE_NAME); - pairedElement.addCollectionElement(datasetElementReverse); - - description.addDatasetElement(pairedElement); + try { + CollectionDescription description = new CollectionDescription(); + description.setCollectionType(DatasetCollectionType.LIST_PAIRED.toString()); + description.setName(COLLECTION_NAME_PAIRED); + + IridaTemporaryFile iridaTemporaryFileForward; + IridaTemporaryFile iridaTemporaryFileReverse; + + Map samplesMapPairForward = new HashMap<>(); + Map samplesMapPairReverse = new HashMap<>(); + Set pathsToUpload = new HashSet<>(); + for (Sample sample : sampleSequenceFilesPaired.keySet()) { + SequenceFilePair sequenceFilePair = sampleSequenceFilesPaired.get(sample); + SequenceFile fileForward = sequenceFilePair.getForwardSequenceFile(); + SequenceFile fileReverse = sequenceFilePair.getReverseSequenceFile(); + + iridaTemporaryFileForward = iridaFileStorageUtility.getTemporaryFile(fileForward.getFile(), "analysis"); + iridaTemporaryFileReverse = iridaFileStorageUtility.getTemporaryFile(fileReverse.getFile(), "analysis"); + + filesToCleanUp.add(iridaTemporaryFileForward); + filesToCleanUp.add(iridaTemporaryFileReverse); + + samplesMapPairForward.put(sample, iridaTemporaryFileForward.getFile()); + samplesMapPairReverse.put(sample, iridaTemporaryFileReverse.getFile()); + pathsToUpload.add(iridaTemporaryFileForward.getFile()); + pathsToUpload.add(iridaTemporaryFileReverse.getFile()); } - } - if(filesToCleanUp.size() > 0) { - try { + // upload files to library and then to a history + Map pathHistoryDatasetId = galaxyHistoriesService.filesToLibraryToHistory(pathsToUpload, + workflowHistory, workflowLibrary, dataStorageType); + + for (Sample sample : sampleSequenceFilesPaired.keySet()) { + Path fileForward = samplesMapPairForward.get(sample); + Path fileReverse = samplesMapPairReverse.get(sample); + + if (!pathHistoryDatasetId.containsKey(fileForward)) { + throw new UploadException("Error, no corresponding history item found for " + fileForward); + } else if (!pathHistoryDatasetId.containsKey(fileReverse)) { + throw new UploadException("Error, no corresponding history item found for " + fileReverse); + } else { + String datasetHistoryIdForward = pathHistoryDatasetId.get(fileForward); + String datasetHistoryIdReverse = pathHistoryDatasetId.get(fileReverse); + + CollectionElement pairedElement = new CollectionElement(); + pairedElement.setName(sample.getSampleName()); + pairedElement.setCollectionType(DatasetCollectionType.PAIRED.toString()); + + HistoryDatasetElement datasetElementForward = new HistoryDatasetElement(); + datasetElementForward.setId(datasetHistoryIdForward); + datasetElementForward.setName(FORWARD_NAME); + pairedElement.addCollectionElement(datasetElementForward); + + HistoryDatasetElement datasetElementReverse = new HistoryDatasetElement(); + datasetElementReverse.setId(datasetHistoryIdReverse); + datasetElementReverse.setName(REVERSE_NAME); + pairedElement.addCollectionElement(datasetElementReverse); + + description.addDatasetElement(pairedElement); + } + } + return galaxyHistoriesService.constructCollection(description, workflowHistory); + } finally { + if (filesToCleanUp.size() > 0) { cleanupTemporaryGalaxyFiles(filesToCleanUp); - } catch (StorageException e) { - throw new StorageException(e.getMessage()); - } catch (IOException e) { - throw new IOException(e.getMessage()); } } - return galaxyHistoriesService.constructCollection(description, workflowHistory); } /** - * Clean up any temporary files that were downloaded and uploaded to Galaxy + * Clean up a collection of temporary files that were downloaded and uploaded to Galaxy * from an object store * * @param filesToCleanUp A list of {@link IridaTemporaryFile}'s to cleanup. - * @throws IOException If there was an error cleaning up the temporary file */ - private void cleanupTemporaryGalaxyFiles(List filesToCleanUp) throws IOException { + private void cleanupTemporaryGalaxyFiles(List filesToCleanUp) { for (IridaTemporaryFile itf : filesToCleanUp) { /* * If there was an error when cleaning up the downloaded temporary files then throw an exception @@ -232,7 +222,6 @@ private void cleanupTemporaryGalaxyFiles(List filesToCleanUp try { iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(itf); } catch (StorageException e) { - Files.delete(itf.getFile()); throw new StorageException(e.getMessage()); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java index c5f458122b9..c50fd8692ed 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisWorkspaceServiceGalaxy.java @@ -301,26 +301,24 @@ private void prepareReferenceFile(ReferenceFile referenceFile, History workflowH WorkflowDetails workflowDetails, WorkflowInvocationInputs inputs) throws UploadException, GalaxyDatasetException, WorkflowException, IOException { - IridaTemporaryFile iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(referenceFile.getFile()); + IridaTemporaryFile iridaTemporaryFile = null; + try { + iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(referenceFile.getFile()); - Dataset referenceDataset = galaxyHistoriesService.fileToHistory(iridaTemporaryFile.getFile(), InputFileType.FASTA, - workflowHistory); + Dataset referenceDataset = galaxyHistoriesService.fileToHistory(iridaTemporaryFile.getFile(), InputFileType.FASTA, + workflowHistory); - String workflowReferenceFileInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, - referenceFileLabel); + String workflowReferenceFileInputId = galaxyWorkflowService.getWorkflowInputId(workflowDetails, + referenceFileLabel); - inputs.setInput(workflowReferenceFileInputId, - new WorkflowInvocationInputs.WorkflowInvocationInput(referenceDataset.getId(), WorkflowInvocationInputs.InputSourceType.HDA)); + inputs.setInput(workflowReferenceFileInputId, + new WorkflowInvocationInputs.WorkflowInvocationInput(referenceDataset.getId(), + WorkflowInvocationInputs.InputSourceType.HDA)); - /* - * If there was an error when cleaning up the downloaded temporary files then throw an exception - * and try to delete the temporary file again so it is not left behind - */ - try { - iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); - } catch (StorageException e) { - Files.delete(iridaTemporaryFile.getFile()); - throw new StorageException(e.getMessage()); + } finally { + if(iridaTemporaryFile != null) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + } } } From 9e2ce84c606ff33198453f1609b6eff648cf0ffc Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 15 Jul 2021 15:40:10 -0500 Subject: [PATCH 217/655] Removed unused import and updated javadoc --- .../workspace/galaxy/AnalysisCollectionServiceGalaxy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index ed168878e87..bf20a92de94 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -21,7 +21,6 @@ import com.github.jmchilton.blend4j.galaxy.beans.collection.response.CollectionResponse; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.*; @@ -131,6 +130,7 @@ public CollectionResponse uploadSequenceFilesSingleEnd( * given files. * @throws ExecutionManagerException If there was an error uploading the files. * @throws IOException If there was an error reading the sequence file. + * @throws StorageException If there was an error removing a temporary file */ public CollectionResponse uploadSequenceFilesPaired( Map sampleSequenceFilesPaired, History workflowHistory, From 1d083a5375a05f2e726e2d95c00ccfe0413511bb Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 16 Jul 2021 12:30:46 -0500 Subject: [PATCH 218/655] Updated to store exception messages in a list and once the for loop is done executing then throw the exceptions --- .../galaxy/AnalysisCollectionServiceGalaxy.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java index bf20a92de94..1dd890d193e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/analysis/workspace/galaxy/AnalysisCollectionServiceGalaxy.java @@ -214,16 +214,22 @@ public CollectionResponse uploadSequenceFilesPaired( * @param filesToCleanUp A list of {@link IridaTemporaryFile}'s to cleanup. */ private void cleanupTemporaryGalaxyFiles(List filesToCleanUp) { + List exceptions = new ArrayList<>(); for (IridaTemporaryFile itf : filesToCleanUp) { /* - * If there was an error when cleaning up the downloaded temporary files then throw an exception - * and try to delete the temporary file again so it is not left behind + * If there was an error when cleaning up the downloaded temporary + * files then add exception message to the list of exception messages */ try { iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(itf); } catch (StorageException e) { - throw new StorageException(e.getMessage()); + exceptions.add(e.getMessage()); } } + if(exceptions.size() > 0) { + // Throw StorageException if there were any exceptions in the for loop above + String exceptionMsg = String.join("\n", exceptions); + throw new StorageException(exceptionMsg); + } } } \ No newline at end of file From 035af4ed70192c2cdec7d375cae60f22b27b80e3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 9 Aug 2021 15:48:41 -0500 Subject: [PATCH 219/655] Updated chromedriver version --- src/main/webapp/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/package.json b/src/main/webapp/package.json index 4ad0cc0c376..7435dfd02ed 100644 --- a/src/main/webapp/package.json +++ b/src/main/webapp/package.json @@ -105,7 +105,7 @@ "autoprefixer": "^10.2.4", "babel-loader": "^8.2.2", "babel-plugin-import": "^1.13.3", - "chromedriver": "90.0.0", + "chromedriver": "92.0.1", "css-loader": "^5.0.1", "css-minimizer-webpack-plugin": "^1.2.0", "eslint": "^7.22.0", From 2219d34385e8c138c36ec3e8d30b961508067501 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 10 Aug 2021 14:20:55 -0500 Subject: [PATCH 220/655] Added StorageType attribute back into irida file storage utilities as it is used when returning data via rest api --- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 3 +++ .../filesystem/IridaFileStorageAzureUtilityImpl.java | 3 +++ .../filesystem/IridaFileStorageLocalUtilityImpl.java | 5 +++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index b9d06dca7ad..eb946ed2ef6 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -16,6 +16,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -40,6 +41,7 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private String bucketName; private BasicAWSCredentials awsCreds; private AmazonS3 s3; + private StorageType storageType; @Autowired public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { @@ -49,6 +51,7 @@ public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, St .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .build(); this.bucketName = bucketName; + this.storageType = StorageType.AWS; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index afa7af43b25..b0441779a42 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; @@ -38,6 +39,7 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; + private StorageType storageType; @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { @@ -45,6 +47,7 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St .sasToken(sasToken) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); + this.storageType = StorageType.AZURE; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 55ab05406cc..42083a5b611 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -18,6 +18,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; +import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; @@ -30,11 +31,11 @@ @Component public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); - + private StorageType storageType; @Autowired public IridaFileStorageLocalUtilityImpl() { - + this.storageType = StorageType.LOCAL; } /** From db0fc89bb4df07050072e0dd548e0b3e34a0e35b Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 24 Aug 2021 16:06:58 -0500 Subject: [PATCH 221/655] Fixed bug with incorrect files being uploaded into galaxy due to an incorrect file identifier that was being returned in GalaxyLibrariesService --- .../upload/galaxy/GalaxyLibrariesService.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index 8a01462902b..837ff9402d1 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -184,7 +184,19 @@ public String fileToLibrary(Path path, InputFileType fileType, logger.error(message); throw new UploadException(message); } else { - return library.getId(); + // Get a list of the contents of the library + List contentsOfLibrary = librariesClient.getLibraryContents(library.getId()); + /* + Get the LibraryContent object (which matches the file name in the path) + that extends the GalaxyObject and from there return the identifier + */ + return contentsOfLibrary.stream() + .filter(f -> f.getName() + .equals("/" + path.getFileName() + .toString())) + .findFirst() + .get() + .getId(); } } } catch (RuntimeException e) { From 3c3c748946d7e1ae333646bcde747f13137c59d3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Aug 2021 11:37:22 -0500 Subject: [PATCH 222/655] Updated azure-storage-blob to latest stable version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 065804f0c39..7b6250b1741 100644 --- a/pom.xml +++ b/pom.xml @@ -1227,7 +1227,7 @@ 1.3 1.2.0 2.1.0 - 12.6.0 + 12.13.0 1.11.894 4.5.12 From 886e8ce910b7706d857ee5d0a4d9bd1dde0bbedc Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Aug 2021 12:59:07 -0500 Subject: [PATCH 223/655] Simplified getting of upload identifier from galaxy --- .../upload/galaxy/GalaxyLibrariesService.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java index 837ff9402d1..a2a9bc70d10 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/upload/galaxy/GalaxyLibrariesService.java @@ -184,19 +184,9 @@ public String fileToLibrary(Path path, InputFileType fileType, logger.error(message); throw new UploadException(message); } else { - // Get a list of the contents of the library - List contentsOfLibrary = librariesClient.getLibraryContents(library.getId()); - /* - Get the LibraryContent object (which matches the file name in the path) - that extends the GalaxyObject and from there return the identifier - */ - return contentsOfLibrary.stream() - .filter(f -> f.getName() - .equals("/" + path.getFileName() - .toString())) - .findFirst() - .get() - .getId(); + List> entity = clientResponse.getEntity(List.class); + return entity.get(0) + .get("id"); } } } catch (RuntimeException e) { From 44bd1cda83c6fcb1d048c592bc5031e1809ab442 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Aug 2021 14:32:11 -0500 Subject: [PATCH 224/655] Updated getting of file size for a cloud file on pipeline page --- .../ria/web/services/UIPipelineService.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java index 7c0922e0523..f958a7b73e8 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java @@ -46,6 +46,7 @@ import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; import ca.corefacility.bioinformatics.irida.service.workflow.WorkflowNamedParametersService; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.github.jmchilton.blend4j.galaxy.beans.TabularToolDataTable; @@ -452,16 +453,12 @@ private List getReferenceFilesForPipeline(List project .map(project -> { List list = new ArrayList<>(); for (Join projectReferenceFileJoin : referenceFileService.getReferenceFilesForProject( - project)) { - try { - ReferenceFile file = projectReferenceFileJoin.getObject(); - Path path = file.getFile(); - String filesize = FileUtilities.humanReadableByteCount(Files.size(path), true); - UIReferenceFile uiReferenceFile = new UIReferenceFile(projectReferenceFileJoin, filesize); - list.add(uiReferenceFile); - } catch (IOException e) { - logger.error(e.getMessage()); - } + project)) { + ReferenceFile file = projectReferenceFileJoin.getObject(); + Path path = file.getFile(); + String filesize = FileUtilities.humanReadableByteCount(IridaFiles.getFileSizeBytes(path), true); + UIReferenceFile uiReferenceFile = new UIReferenceFile(projectReferenceFileJoin, filesize); + list.add(uiReferenceFile); } return list; }) From 9f8363a40147e8dfb509e62b08c38d81703f3bee Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Aug 2021 14:40:40 -0500 Subject: [PATCH 225/655] Updated to use getFileSize method within the ReferenceFile object --- .../irida/ria/web/services/UIPipelineService.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java index f958a7b73e8..cc1831afefb 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java @@ -1,8 +1,5 @@ package ca.corefacility.bioinformatics.irida.ria.web.services; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -46,7 +43,6 @@ import ca.corefacility.bioinformatics.irida.service.ReferenceFileService; import ca.corefacility.bioinformatics.irida.service.workflow.IridaWorkflowsService; import ca.corefacility.bioinformatics.irida.service.workflow.WorkflowNamedParametersService; -import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.github.jmchilton.blend4j.galaxy.beans.TabularToolDataTable; @@ -455,8 +451,7 @@ private List getReferenceFilesForPipeline(List project for (Join projectReferenceFileJoin : referenceFileService.getReferenceFilesForProject( project)) { ReferenceFile file = projectReferenceFileJoin.getObject(); - Path path = file.getFile(); - String filesize = FileUtilities.humanReadableByteCount(IridaFiles.getFileSizeBytes(path), true); + String filesize = file.getFileSize(); UIReferenceFile uiReferenceFile = new UIReferenceFile(projectReferenceFileJoin, filesize); list.add(uiReferenceFile); } From 061e1164cf8d0e0a5e6e2291ebca2c4247c6fc73 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 25 Aug 2021 17:06:31 -0500 Subject: [PATCH 226/655] Updated logic to log an error if a referencefile file cannot be found on server --- .../irida/ria/web/services/UIPipelineService.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java index cc1831afefb..7ef8db6fce7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIPipelineService.java @@ -27,7 +27,6 @@ import ca.corefacility.bioinformatics.irida.model.workflow.submission.IridaWorkflowNamedParameters; import ca.corefacility.bioinformatics.irida.pipeline.results.AnalysisSubmissionSampleProcessor; import ca.corefacility.bioinformatics.irida.pipeline.upload.galaxy.GalaxyToolDataService; -import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.pipeline.SavePipelineParametersRequest; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.pipeline.SavedPipelineParameters; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.references.UIReferenceFile; @@ -452,8 +451,12 @@ private List getReferenceFilesForPipeline(List project project)) { ReferenceFile file = projectReferenceFileJoin.getObject(); String filesize = file.getFileSize(); - UIReferenceFile uiReferenceFile = new UIReferenceFile(projectReferenceFileJoin, filesize); - list.add(uiReferenceFile); + if(!filesize.equals("N/A")) { + UIReferenceFile uiReferenceFile = new UIReferenceFile(projectReferenceFileJoin, filesize); + list.add(uiReferenceFile); + } else { + logger.error("Unable to locate reference file " + file.getLabel()); + } } return list; }) From a256989b5b322f9e57d2818a7127f21d99ea05d3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Sep 2021 10:27:17 -0500 Subject: [PATCH 227/655] Updated try-with-resources blocks to include outputstream for the view classes. Removed method to get analysis output files contents in the rest api as it is being updated in another PR. Moved setting of storagetype in iridafilestorageutility classes into private final variables from the constructor --- .../irida/model/assembly/GenomeAssembly.java | 8 +++---- .../model/sequenceFile/SequenceFile.java | 1 + .../IridaFileStorageAwsUtilityImpl.java | 3 +-- .../IridaFileStorageAzureUtilityImpl.java | 3 +-- .../IridaFileStorageLocalUtilityImpl.java | 3 +-- .../api/RESTAnalysisSubmissionController.java | 22 ------------------- .../irida/web/spring/view/CSVView.java | 9 +++----- .../irida/web/spring/view/FastaView.java | 7 +++--- .../irida/web/spring/view/FastqView.java | 6 ++--- .../irida/web/spring/view/GenbankView.java | 4 +--- .../irida/web/spring/view/NewickFileView.java | 4 +--- 11 files changed, 18 insertions(+), 52 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index ab16d1578af..dcc10be4d93 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -80,14 +80,14 @@ public Date getCreatedDate() { } /** - * Get the size of the genome assembly files + * Get the size of the genome assembly files in bytes * * @return file size * @throws IOException if the file cannot be read */ @JsonIgnore - public long getFileSizeBytes() throws IOException { - return Files.size(getFile()); + public Long getFileSizeBytes() throws IOException { + return IridaFiles.getFileSizeBytes(getFile()); } /** @@ -102,7 +102,7 @@ public void addSampleGenomeAssemblyJoin(SampleGenomeAssemblyJoin join) { } /** - * Gets the assembly file size. + * Gets the assembly file size as a human readable string. * * @return The assembly file size. */ diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 91bf0243d09..b6874db4224 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -314,6 +314,7 @@ public InputStream getFileInputStream() { * * @return if file exists or not */ + @JsonIgnore public Long getFileSizeBytes() { return IridaFiles.getFileSizeBytes(getFile()); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index eb946ed2ef6..bff63a329c4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -41,7 +41,7 @@ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { private String bucketName; private BasicAWSCredentials awsCreds; private AmazonS3 s3; - private StorageType storageType; + private final StorageType storageType = StorageType.AWS; @Autowired public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, String accessKey, String secretKey) { @@ -51,7 +51,6 @@ public IridaFileStorageAwsUtilityImpl(String bucketName, String bucketRegion, St .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .build(); this.bucketName = bucketName; - this.storageType = StorageType.AWS; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index b0441779a42..7ad20fcc308 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -39,7 +39,7 @@ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility private BlobServiceClient blobServiceClient; private BlobContainerClient containerClient; - private StorageType storageType; + private final StorageType storageType = StorageType.AZURE; @Autowired public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, String containerName) { @@ -47,7 +47,6 @@ public IridaFileStorageAzureUtilityImpl(String containerUrl, String sasToken, St .sasToken(sasToken) .buildClient(); this.containerClient = blobServiceClient.getBlobContainerClient(containerName); - this.storageType = StorageType.AZURE; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 42083a5b611..8edc056f941 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -31,11 +31,10 @@ @Component public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { private static final Logger logger = LoggerFactory.getLogger(IridaFileStorageLocalUtilityImpl.class); - private StorageType storageType; + private final StorageType storageType = StorageType.LOCAL; @Autowired public IridaFileStorageLocalUtilityImpl() { - this.storageType = StorageType.LOCAL; } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index 2e032800421..2aa6379b309 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -318,28 +318,6 @@ public ResponseResource getAnalysisOutputFile(@PathVariable return responseObject; } - /** - * Get the actual file contents for an analysis output file. - * - * @param submissionId The {@link AnalysisSubmission} id - * @param fileId The {@link AnalysisOutputFile} id - * @return a {@link ByteArrayResource} containing the contents of the {@link AnalysisOutputFile}. - * @throws IOException if the file input stream cannot be read - */ - @Operation(operationId = "getAnalysisOutputFileContents", summary = "Get the analysis output file contents", description = "Get the analysis output file contents for the given analysis submission.", tags = "analysisSubmissions") - @RequestMapping(value = "/{submissionId}/analysis/file/{fileId}", produces = MediaType.TEXT_PLAIN_VALUE) - @ResponseBody - public ResponseResource getAnalysisOutputFileContents(@PathVariable Long submissionId, - @PathVariable Long fileId) throws IOException { - AnalysisOutputFile analysisOutputFile = getOutputFileForSubmission(submissionId, fileId); - try (InputStream inputStream = analysisOutputFile.getFileInputStream()) { - ByteArrayResource fileContents = new ByteArrayResource(inputStream.readAllBytes()); - return new ResponseResource<>(fileContents); - } catch (IOException e) { - throw new IOException("Unable to read input stream ", e); - } - } - /** * Get the {@link AnalysisOutputFile} for an {@link AnalysisSubmission} and given file ID * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java index b84b9b56d48..1bc3d00e712 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/CSVView.java @@ -23,8 +23,7 @@ /** * Write out CSV files to the client. */ -public class -CSVView extends AbstractView { +public class CSVView extends AbstractView { public static final String DEFAULT_CONTENT_TYPE = "text/csv"; private static final Logger logger = LoggerFactory.getLogger(CSVView.class); @@ -51,12 +50,10 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - try(InputStream is = sfr.getFileInputStream()) { - OutputStream os = response.getOutputStream(); + try (InputStream is = sfr.getFileInputStream(); OutputStream os = response.getOutputStream();) { IOUtils.copy(is, os); os.flush(); - os.close(); - }catch (IOException e) { + } catch (IOException e) { throw new IOException("Unable to read inputstream ", e); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java index c32a0778159..e0e9cd3931d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastaView.java @@ -55,12 +55,11 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(IridaFiles.getFileSizeBytes(fileContent))); - try(InputStream is = IridaFiles.getFileInputStream(fileContent)) { - OutputStream os = response.getOutputStream(); + try (InputStream is = IridaFiles.getFileInputStream(fileContent); + OutputStream os = response.getOutputStream();) { IOUtils.copy(is, os); os.flush(); - os.close(); - }catch (IOException e) { + } catch (IOException e) { throw new IOException("Unable to read inputstream ", e); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java index d219eb375f1..cabf1beb34c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/FastqView.java @@ -51,12 +51,10 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - try(InputStream is = sfr.getFileInputStream()) { - OutputStream os = response.getOutputStream(); + try (InputStream is = sfr.getFileInputStream(); OutputStream os = response.getOutputStream();) { IOUtils.copy(is, os); os.flush(); - os.close(); - }catch (IOException e) { + } catch (IOException e) { throw new IOException("Unable to read inputstream ", e); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java index f1e0a73e3d1..37ad0af034b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/GenbankView.java @@ -51,11 +51,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - try(InputStream is = sfr.getFileInputStream()) { - OutputStream os = response.getOutputStream(); + try(InputStream is = sfr.getFileInputStream(); OutputStream os = response.getOutputStream();) { IOUtils.copy(is, os); os.flush(); - os.close(); }catch (IOException e) { throw new IOException("Unable to read inputstream ", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java index ace53f7f201..e1bec359485 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/spring/view/NewickFileView.java @@ -50,11 +50,9 @@ protected void renderMergedOutputModel(Map model, HttpServletReq response.setHeader(HttpHeaders.CONTENT_TYPE, DEFAULT_CONTENT_TYPE); response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(sfr.getFileSizeBytes())); - try(InputStream is = sfr.getFileInputStream()) { - OutputStream os = response.getOutputStream(); + try(InputStream is = sfr.getFileInputStream(); OutputStream os = response.getOutputStream();) { IOUtils.copy(is, os); os.flush(); - os.close(); }catch (IOException e) { throw new IOException("Unable to read inputstream ", e); } From 53eba47c028643f33dfce5482a945e1ca87fce47 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Sep 2021 11:32:39 -0500 Subject: [PATCH 228/655] Updated failing test --- .../unit/web/services/UIAnalysesOutputsServiceTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java index c11c7d5d9e9..ed8ce3e45a0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java @@ -6,7 +6,11 @@ import org.junit.Before; import org.junit.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.ProjectSampleAnalysisOutputInfo; import ca.corefacility.bioinformatics.irida.ria.web.components.AnalysisOutputFileDownloadManager; import ca.corefacility.bioinformatics.irida.ria.web.services.UIAnalysesOutputsService; @@ -61,6 +65,10 @@ public void getAutomatedSingleAnalysisOutputs() { @Test public void getUserSingleAnalysisOutputs() { + User user = userService.getUserByUsername(principal.getName()); + Authentication auth = new UsernamePasswordAuthenticationToken(user, null); + SecurityContextHolder.getContext().setAuthentication(auth); + List userProjectSampleAnalysisOutputInfos = uiProjectAnalysesService.getUserSingleSampleOutputs(); verify(userService, times(1)).getUserByUsername(principal.getName()); From 909182cdfb4700dd623b7593920752d4f12a36a6 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 13 Sep 2021 11:36:29 -0500 Subject: [PATCH 229/655] Removed unused imports --- .../bioinformatics/irida/model/assembly/GenomeAssembly.java | 1 - .../web/controller/api/RESTAnalysisSubmissionController.java | 4 ---- 2 files changed, 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index dcc10be4d93..004c67dbe0d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Date; import java.util.List; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index 2aa6379b309..3b2dd9f0485 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -28,16 +28,12 @@ import io.swagger.v3.oas.annotations.Operation; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.InputStream; import java.util.*; import java.util.stream.Collectors; From 52f90ce59c2354e253502401626d3591ccd7c9f4 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 27 Oct 2021 12:56:20 -0500 Subject: [PATCH 230/655] Fixed broken test --- .../ria/unit/web/services/UIAnalysesOutputsServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java index f8f9d749486..4afc7b3e595 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java @@ -88,7 +88,7 @@ public void getUserSingleAnalysisOutputs() { List userProjectSampleAnalysisOutputInfos = uiProjectAnalysesService.getUserSingleSampleOutputs(); - verify(userService, times(1)).getUserByUsername(principal.getName()); + verify(userService, times(2)).getUserByUsername(principal.getName()); verify(analysisSubmissionService, times(1)).getAllUserAnalysisOutputInfo( userService.getUserByUsername(principal.getName())); From f84c7134be724244e2a08bdf7972f86bc139bc75 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Wed, 27 Oct 2021 16:53:56 -0500 Subject: [PATCH 231/655] updated getAnalysisOutputFileContents method to return inputstream to file --- .../api/RESTAnalysisSubmissionController.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java index 7c0c2501506..d587f042a18 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/RESTAnalysisSubmissionController.java @@ -29,7 +29,7 @@ import io.swagger.v3.oas.annotations.Operation; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.InputStreamResource; import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; @@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletResponse; +import java.io.InputStream; import java.util.*; import java.util.stream.Collectors; @@ -322,16 +323,21 @@ public ResponseResource getAnalysisOutputFile(@PathVariable * * @param submissionId The {@link AnalysisSubmission} id * @param fileId The {@link AnalysisOutputFile} id - * @return a {@link FileSystemResource} containing the contents of the {@link AnalysisOutputFile}. + * @return a {@link InputStreamResource} containing the {@link InputStream} of the {@link AnalysisOutputFile}. */ @Hidden @RequestMapping(value = "/{submissionId}/analysis/file/{fileId}", produces = MediaType.TEXT_PLAIN_VALUE) @ResponseBody - public FileSystemResource getAnalysisOutputFileContents(@PathVariable Long submissionId, + public InputStreamResource getAnalysisOutputFileContents(@PathVariable Long submissionId, @PathVariable Long fileId) { AnalysisOutputFile analysisOutputFile = getOutputFileForSubmission(submissionId, fileId); - return new FileSystemResource(analysisOutputFile.getFile() - .toFile()); + /* + The inputstream is closed in the underlying class (ResourceHttpMessageConverter) so we don't need to close it here. + See below links for more details + https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/http/converter/ResourceHttpMessageConverter.html + https://github.com/spring-projects/spring-framework/blob/main/spring-web/src/main/java/org/springframework/http/converter/ResourceHttpMessageConverter.java#L139 + */ + return new InputStreamResource(analysisOutputFile.getFileInputStream()); } /** From c5f1029d72d2a729c8e2289a1eca09deee3b9928 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 8 Feb 2022 13:33:47 -0600 Subject: [PATCH 232/655] Fixed test error --- .../irida/processing/impl/ChecksumFileProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java index f556133f743..7922395a224 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/impl/ChecksumFileProcessor.java @@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.processing.FileProcessor; @@ -59,7 +60,7 @@ public void process(SequencingObject sequencingObject) { logger.trace("Checksum generated for file " + file.getId() + ": " + shaDigest); file.setUploadSha256(shaDigest); fileRepository.saveMetadata(file); - } catch (IOException e) { + } catch (IOException | StorageException e) { throw new FileProcessorException("could not calculate checksum", e); } From 614b67e95e00ec22b79dfe8327ba654bd0051dfe Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 8 Feb 2022 13:53:20 -0600 Subject: [PATCH 233/655] Removed unused imports --- .../pipeline/results/updater/impl/SISTRSampleUpdater.java | 2 -- .../filesystem/IridaFileStorageAwsUtilityImpl.java | 5 +---- .../filesystem/IridaFileStorageAzureUtilityImpl.java | 5 +---- .../filesystem/IridaFileStorageLocalUtilityImpl.java | 5 +---- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java index 0eb99f97145..3ef42baf8f3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/SISTRSampleUpdater.java @@ -21,13 +21,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import org.apache.jena.atlas.io.IO; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.IOException; import java.io.InputStream; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.*; diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java index 62616e68829..0b3d77db679 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java @@ -13,8 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -24,7 +22,6 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.GetObjectRequest; @@ -32,7 +29,7 @@ import com.amazonaws.services.s3.model.S3ObjectInputStream; /** - * Component implementation of file utilities for aws storage + * Implementation of file utilities for aws storage */ public class IridaFileStorageAwsUtilityImpl implements IridaFileStorageUtility { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java index 0694d006fdd..f3db8bb8bc0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java @@ -18,9 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; -import org.springframework.stereotype.Service; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; @@ -32,7 +29,7 @@ import com.azure.storage.blob.specialized.BlobInputStream; /** - * Component implementation of file utilities for azure storage + * Implementation of file utilities for azure storage */ public class IridaFileStorageAzureUtilityImpl implements IridaFileStorageUtility { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 9a1acd17293..e943e282646 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -15,18 +15,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.processing.FileProcessorException; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; /** - * Component implementation of file utilities for local storage + * Implementation of file utilities for local storage */ public class IridaFileStorageLocalUtilityImpl implements IridaFileStorageUtility { From f648d5fee66371e918390ef7c82f54f51477fc68 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 8 Feb 2022 14:48:10 -0600 Subject: [PATCH 234/655] Fixed test --- .../irida/ria/unit/utilities/FileUtilitiesTest.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java index f6b1c7f92fb..86e8e0ec331 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java @@ -7,12 +7,23 @@ import java.nio.file.Path; import java.nio.file.Paths; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; public class FileUtilitiesTest { - + private IridaFileStorageUtility iridaFileStorageUtility; + + @BeforeEach + public void setup () { + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + } + @Test public void testIsZippedFile() throws IOException { Path snpTreePath = Paths.get("src/test/resources/files/snp_tree.tree"); From 0ad4610f6c3b9e947a62cb927f466d5f7c01ec63 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 10 Feb 2022 16:53:33 -0600 Subject: [PATCH 235/655] Updated htmloutput to use iridafilestorageutility. Moved readchunk from fileutilities into iridafilestoragelocalutility as it's the only place it is called from. --- .../IridaFileStorageLocalUtilityImpl.java | 11 ++++++-- .../irida/ria/utilities/FileUtilities.java | 28 ++----------------- .../web/analysis/AnalysisAjaxController.java | 3 +- .../ria/web/analysis/AnalysisController.java | 23 +++++++++++---- .../web/analysis/AnalysisControllerTest.java | 6 +++- 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index c87de1e4a46..4679684f6e3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -20,7 +21,6 @@ import ca.corefacility.bioinformatics.irida.model.enums.StorageType; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; -import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse; /** @@ -237,7 +237,14 @@ public FileChunkResponse readChunk(Path file, Long seek, Long chunk) { try { final RandomAccessFile randomAccessFile = new RandomAccessFile(file.toFile(), "r"); randomAccessFile.seek(seek); - return new FileChunkResponse(FileUtilities.readChunk(randomAccessFile, seek, chunk), randomAccessFile.getFilePointer()); + String chunkResponse = ""; + byte[] bytes = new byte[Math.toIntExact(chunk)]; + final int bytesRead = randomAccessFile.read(bytes); + if (bytesRead > -1) { + chunkResponse = new String(bytes, 0, bytesRead, Charset.defaultCharset()); + } + + return new FileChunkResponse(chunkResponse, randomAccessFile.getFilePointer()); } catch (IOException e ) { logger.error("Could not read output file ", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java index b444dcc7dca..26b401f8951 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/FileUtilities.java @@ -1,7 +1,6 @@ package ca.corefacility.bioinformatics.irida.ria.utilities; import java.io.*; -import java.nio.charset.Charset; import java.nio.file.Path; @@ -22,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.ProjectSampleAnalysisOutputInfo; @@ -286,25 +286,6 @@ public static String getUniqueFilename(Path filePath, String sampleName, Long sa return filename; } - /** - * Read bytes of length {@code chunk} of a file starting at byte {@code seek}. - * - * @param raf File reader - * @param seek FilePointer position to start reading at - * @param chunk Number of bytes to read from file - * @return Chunk of file as String - * @throws IOException if error enountered while reading file - */ - public static String readChunk(RandomAccessFile raf, Long seek, Long chunk) throws IOException { - raf.seek(seek); - byte[] bytes = new byte[Math.toIntExact(chunk)]; - final int bytesRead = raf.read(bytes); - if (bytesRead == -1) { - return ""; - } - return new String(bytes, 0, bytesRead, Charset.defaultCharset()); - } - /** * Read a specified number of lines from a file. * @@ -356,10 +337,7 @@ public static List readLinesFromFilePointer(RandomAccessFile randomAcces * @return parsed excel file data */ public static ExcelData parseExcelFile(AnalysisOutputFile outputFile, int sheetIndex) { - try { - InputStream is = new FileInputStream(new File(outputFile.getFile() - .toAbsolutePath() - .toString())); + try(InputStream is = outputFile.getFileInputStream()) { Workbook workbook = StreamingReader.builder() .open(is); @@ -411,7 +389,7 @@ public static ExcelData parseExcelFile(AnalysisOutputFile outputFile, int sheetI } } return new ExcelData(headers, excelRows, excelSheetNames, false); - } catch (IOException e) { + } catch (IOException | StorageException e) { logger.error("Error opening file" + outputFile.getLabel()); } // Should only reach here if the file could not be opened diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index 40ef5dad667..27ad3222fb4 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -700,8 +700,7 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { Path path = null; if (analysis.getAnalysisOutputFile(sistrFileKey) != null) { path = analysis.getAnalysisOutputFile(sistrFileKey).getFile(); - - try(InputStream inputStream = iridaFileStorageUtility.getFileInputStream(path)) { + try(InputStream inputStream = analysis.getAnalysisOutputFile(sistrFileKey).getFileInputStream()) { String json = new Scanner(inputStream).useDelimiter("\\Z") .next(); // verify file is proper json file and map to a SistrResult list diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisController.java index be8704cb8a8..40d81ed11ee 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisController.java @@ -25,6 +25,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.user.Role; import ca.corefacility.bioinformatics.irida.model.user.User; import ca.corefacility.bioinformatics.irida.model.workflow.IridaWorkflow; @@ -32,6 +33,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.analysis.AnalysisOutputFile; import ca.corefacility.bioinformatics.irida.model.workflow.analysis.type.AnalysisType; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaTemporaryFile; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; import ca.corefacility.bioinformatics.irida.service.user.UserService; @@ -60,15 +63,18 @@ public class AnalysisController { private IridaWorkflowsService workflowsService; private UserService userService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @Autowired public AnalysisController(AnalysisSubmissionService analysisSubmissionService, - IridaWorkflowsService iridaWorkflowsService, UserService userService, MessageSource messageSource) { + IridaWorkflowsService iridaWorkflowsService, UserService userService, MessageSource messageSource, + IridaFileStorageUtility iridaFileStorageUtility) { this.analysisSubmissionService = analysisSubmissionService; this.workflowsService = iridaWorkflowsService; this.userService = userService; this.messageSource = messageSource; + this.iridaFileStorageUtility = iridaFileStorageUtility; } // ************************************************************************************************ @@ -194,9 +200,10 @@ public void getHtmlOutputForSubmission(@PathVariable Long submissionId, @Request // Set the common Http headers response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "inline"); response.setHeader(HttpHeaders.CONTENT_TYPE, "text/html"); - + IridaTemporaryFile iridaTemporaryFile = null; if (zipped) { - String htmlFile = outputFile.getFile().toFile().toString(); + iridaTemporaryFile = iridaFileStorageUtility.getTemporaryFile(outputFile.getFile()); + String htmlFile = iridaTemporaryFile.getFile().toFile().toString(); if (htmlFile.endsWith(".html.zip")) { htmlFile = htmlFile.substring(0, htmlFile.length()-4); } @@ -235,14 +242,18 @@ public void getHtmlOutputForSubmission(@PathVariable Long submissionId, @Request outputStream.write(htmlOutputNotFound.getBytes(StandardCharsets.UTF_8)); outputStream.flush(); outputStream.close(); + } finally { + if(iridaTemporaryFile != null) { + iridaFileStorageUtility.cleanupDownloadedLocalTemporaryFiles(iridaTemporaryFile); + } } } else { - try (InputStream inputStream = new FileInputStream(outputFile.getFile() - .toString()); OutputStream outputStream = response.getOutputStream()) { + try (InputStream inputStream = outputFile.getFileInputStream(); + OutputStream outputStream = response.getOutputStream()) { // Copy the file contents to the response outputstream IOUtils.copy(inputStream, outputStream); - } catch (IOException e) { + } catch (IOException | StorageException e) { logger.debug("Html output not found."); String htmlOutputNotFound = messageSource.getMessage("analysis.html.file.not.found", new Object[] { filename }, locale); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisControllerTest.java index 44093288608..6af891395c0 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/analysis/AnalysisControllerTest.java @@ -17,6 +17,8 @@ import ca.corefacility.bioinformatics.irida.model.workflow.description.IridaWorkflowInput; import ca.corefacility.bioinformatics.irida.model.workflow.structure.IridaWorkflowStructure; import ca.corefacility.bioinformatics.irida.model.workflow.submission.AnalysisSubmission; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.unit.TestDataFactory; import ca.corefacility.bioinformatics.irida.ria.web.analysis.AnalysisController; import ca.corefacility.bioinformatics.irida.service.AnalysisSubmissionService; @@ -48,6 +50,7 @@ public class AnalysisControllerTest { private UserService userServiceMock; private AnalysisTypesService analysisTypesService; private MessageSource messageSource; + private IridaFileStorageUtility iridaFileStorageUtility; @BeforeEach public void init() { @@ -56,9 +59,10 @@ public void init() { userServiceMock = mock(UserService.class); analysisTypesService = mock(AnalysisTypesService.class); messageSource = mock(MessageSource.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); analysisController = new AnalysisController(analysisSubmissionServiceMock, iridaWorkflowsServiceMock, - userServiceMock, messageSource); + userServiceMock, messageSource, iridaFileStorageUtility); } From 8ee8d890d9ae5084b3890ecf30a6e03d573ce411 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 10 Feb 2022 17:01:57 -0600 Subject: [PATCH 236/655] Removed setting of user and authentication as it is already done in beforeach during test setup --- .../ria/unit/web/services/UIAnalysesOutputsServiceTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java index 2a1f566da5d..2b32cfda307 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIAnalysesOutputsServiceTest.java @@ -82,13 +82,9 @@ public void getAutomatedSingleAnalysisOutputs() { @Test public void getUserSingleAnalysisOutputs() { - User user = userService.getUserByUsername(principal.getName()); - Authentication auth = new UsernamePasswordAuthenticationToken(user, null); - SecurityContextHolder.getContext().setAuthentication(auth); - List userProjectSampleAnalysisOutputInfos = uiProjectAnalysesService.getUserSingleSampleOutputs(); - verify(userService, times(2)).getUserByUsername(principal.getName()); + verify(userService, times(1)).getUserByUsername(principal.getName()); verify(analysisSubmissionService, times(1)).getAllUserAnalysisOutputInfo( userService.getUserByUsername(principal.getName())); From 48e3469457d51b5f5471c8d88d0e0b17ab955d6e Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 18 Feb 2022 21:39:39 -0600 Subject: [PATCH 237/655] Added column to sequence file to store location of file (local, azure, aws, etc) --- .../model/sequenceFile/SequenceFile.java | 8 +++++++- .../sequencefile/SequenceFileResource.java | 2 ++ .../RESTSampleSequenceFilesController.java | 3 +++ .../irida/database/all-changes.xml | 1 + ...add-storage-type-to-sequencing-objects.xml | 19 +++++++++++++++++++ .../database/changesets/22.05/all-changes.xml | 9 +++++++++ 6 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java index 8ffc9e80477..8f19c2dbf47 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/SequenceFile.java @@ -75,6 +75,8 @@ public class SequenceFile extends IridaRepresentationModel private Long fileRevisionNumber; // the filesystem file revision number + @Column(name = "storage_type") + private String storageType; // Key/value map of additional properties you could set on a sequence file. // This may contain optional sequencer specific properties. @@ -119,6 +121,7 @@ public String getLabel() { public SequenceFile(Path sampleFile) { this(); this.file = sampleFile; + this.storageType = IridaFiles.getStorageType(); } @Override @@ -320,7 +323,10 @@ public Long getFileSizeBytes() { } public String getStorageType(){ - return IridaFiles.getStorageType(); + return storageType; } + public void setStorageType(String storageType) { + this.storageType = storageType; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java index 8d4203a5ee7..5934bd132e3 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/assembler/resource/sequencefile/SequenceFileResource.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.web.assembler.resource.sequencefile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -21,6 +22,7 @@ public class SequenceFileResource { public SequenceFileResource() { resource = new SequenceFile(); + resource.setStorageType(IridaFiles.getStorageType()); } public SequenceFileResource(SequenceFile sequenceFile) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java index 6136ad257b0..c931ff58db2 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/web/controller/api/samples/RESTSampleSequenceFilesController.java @@ -8,6 +8,7 @@ import javax.servlet.http.HttpServletResponse; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResponseResource; import io.swagger.v3.oas.annotations.Operation; @@ -403,6 +404,7 @@ public ResponseResource addNewSequenceFileToSample(@PathVariable L } sf.setFile(target); + sf.setStorageType(IridaFiles.getStorageType()); SingleEndSequenceFile singleEndSequenceFile = new SingleEndSequenceFile(sf); if (miseqRun != null) { @@ -517,6 +519,7 @@ public ResponseResource addNewFast5FileToSample(@PathVariable Long } sf.setFile(target); + sf.setStorageType(IridaFiles.getStorageType()); Fast5Object fast5Object = new Fast5Object(sf); diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml index e1a896b4286..928e325d4c9 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml @@ -71,4 +71,5 @@ + diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml new file mode 100644 index 00000000000..c737fbcaabd --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + UPDATE sequence_file SET storage_type="local" + UPDATE sequence_file_AUD SET storage_type="local" + + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml new file mode 100644 index 00000000000..d9d4d867c06 --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml @@ -0,0 +1,9 @@ + + + + + From 8f610b730926971eae05af603dedcd100be61e51 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Sat, 19 Feb 2022 00:37:59 -0600 Subject: [PATCH 238/655] Added storage type to genome and uploaded assembly, reference file, and analysis output file --- .../irida/model/assembly/GenomeAssembly.java | 10 +++- .../irida/model/project/ReferenceFile.java | 11 ++++ .../workflow/analysis/AnalysisOutputFile.java | 11 ++++ .../22.05/add-storage-type-to-file-tables.xml | 55 +++++++++++++++++++ ...add-storage-type-to-sequencing-objects.xml | 19 ------- .../database/changesets/22.05/all-changes.xml | 2 +- 6 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-file-tables.xml delete mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java index 82d24efdc34..62b0a7c6cda 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/assembly/GenomeAssembly.java @@ -48,6 +48,9 @@ public abstract class GenomeAssembly extends IridaRepresentationModel @Column(name = "created_date", updatable = false) private Date createdDate; + @Column(name = "storage_type") + private String storageType; + @OneToMany(fetch = FetchType.LAZY, mappedBy = "genomeAssembly") private List sampleGenomeAssemblies; @@ -60,6 +63,7 @@ protected GenomeAssembly() { public GenomeAssembly(Date createdDate) { this.createdDate = createdDate; this.sampleGenomeAssemblies = Lists.newArrayList(); + this.storageType = IridaFiles.getStorageType(); } @Override @@ -155,6 +159,10 @@ public InputStream getFileInputStream() { } public String getStorageType(){ - return IridaFiles.getStorageType(); + return storageType; + } + + public void setStorageType(String storageType) { + this.storageType = storageType; } } \ No newline at end of file diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java index c64da9a9684..d56d478fa4e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/project/ReferenceFile.java @@ -67,6 +67,9 @@ public class ReferenceFile implements VersionedFileFields, MutableIridaThi private Long fileLength; + @Column(name = "storage_type") + private String storageType; + @Override public int hashCode() { return file.hashCode(); @@ -91,6 +94,7 @@ public ReferenceFile() { public ReferenceFile(Path file) { this(); this.file = file; + this.storageType = IridaFiles.getStorageType(); } @Override @@ -165,4 +169,11 @@ public InputStream getFileInputStream() { return IridaFiles.getFileInputStream(file); } + public String getStorageType(){ + return storageType; + } + + public void setStorageType(String storageType) { + this.storageType = storageType; + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java index 0cc86310c59..9adebd3dc5d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/workflow/analysis/AnalysisOutputFile.java @@ -75,6 +75,9 @@ public class AnalysisOutputFile extends IridaRepresentationModel implements Irid @Column(name = "label_prefix") private final String labelPrefix; + @Column(name = "storage_type") + private String storageType; + /** * for hibernate */ @@ -109,6 +112,7 @@ public AnalysisOutputFile(final Path file, final String labelPrefix, final Strin this.executionManagerFileId = executionManagerFileId; this.createdByTool = createdByTool; this.labelPrefix = labelPrefix; + this.storageType = IridaFiles.getStorageType(); } @Override @@ -236,4 +240,11 @@ public Long getFileSizeBytes() { return IridaFiles.getFileSizeBytes(getFile()); } + public String getStorageType(){ + return storageType; + } + + public void setStorageType(String storageType) { + this.storageType = storageType; + } } diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-file-tables.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-file-tables.xml new file mode 100644 index 00000000000..e313b93a8d9 --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-file-tables.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + UPDATE sequence_file SET storage_type="local" + UPDATE sequence_file_AUD SET storage_type="local" + + UPDATE genome_assembly SET storage_type="local" + UPDATE genome_assembly_AUD SET storage_type="local" + + UPDATE uploaded_assembly SET storage_type="local" + UPDATE uploaded_assembly_AUD SET storage_type="local" + + UPDATE analysis_output_file SET storage_type="local" + + UPDATE reference_file SET storage_type="local" + UPDATE reference_file_AUD SET storage_type="local" + + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml deleted file mode 100644 index c737fbcaabd..00000000000 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-sequencing-objects.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - UPDATE sequence_file SET storage_type="local" - UPDATE sequence_file_AUD SET storage_type="local" - - \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml index d9d4d867c06..49e11e311f3 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml @@ -4,6 +4,6 @@ xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd"> - From 21f90300e21bba8d4c2dedc664d19dd9386179b2 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 21 Apr 2022 18:07:59 -0500 Subject: [PATCH 239/655] Fixed null pointer test errors --- .../model/sequenceFile/unit/SequenceFilePairTest.java | 8 ++++++++ .../unit/samples/RESTSampleAssemblyControllerTest.java | 7 +++++++ .../unit/samples/SampleSequenceFilesControllerTest.java | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java index a6c2fb79bde..f4b0aea8430 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/model/sequenceFile/unit/SequenceFilePairTest.java @@ -13,6 +13,9 @@ import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; /** * Tests for SequenceFilePair @@ -35,6 +38,8 @@ public class SequenceFilePairTest { private SequenceFilePair sequenceFilePairGood; private SequenceFilePair sequenceFilePairBad; + private IridaFileStorageUtility iridaFileStorageUtility; + /** * Sets up files for tests. * @@ -42,6 +47,9 @@ public class SequenceFilePairTest { */ @BeforeEach public void setup() throws IOException { + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + Path tempDir = Paths.get("/tmp"); forwardPathGood = tempDir.resolve("Test_R1_001.fastq"); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/RESTSampleAssemblyControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/RESTSampleAssemblyControllerTest.java index 7fa00b2f7ba..b665b5642ca 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/RESTSampleAssemblyControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/RESTSampleAssemblyControllerTest.java @@ -2,6 +2,9 @@ import java.util.List; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResponseResource; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +35,8 @@ public class RESTSampleAssemblyControllerTest { private SampleService sampleService; private GenomeAssemblyService genomeAssemblyService; + private IridaFileStorageUtility iridaFileStorageUtility; + GenomeAssemblyFromAnalysis assemblyFromAnalysis; List assemblies; Sample s1; @@ -40,6 +45,8 @@ public class RESTSampleAssemblyControllerTest { public void setUp() { sampleService = mock(SampleService.class); genomeAssemblyService = mock(GenomeAssemblyService.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); controller = new RESTSampleAssemblyController(sampleService, genomeAssemblyService); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java index 7e43db568e5..64a2704e5c7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/web/controller/test/unit/samples/SampleSequenceFilesControllerTest.java @@ -20,6 +20,9 @@ import ca.corefacility.bioinformatics.irida.model.enums.SequencingRunUploadStatus; import ca.corefacility.bioinformatics.irida.model.run.SequencingRun; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; import ca.corefacility.bioinformatics.irida.web.assembler.resource.ResponseResource; import org.junit.jupiter.api.BeforeEach; @@ -65,6 +68,8 @@ public class SampleSequenceFilesControllerTest { private AnalysisService analysisService; private SequencingRun sequencingRun; + private IridaFileStorageUtility iridaFileStorageUtility; + @BeforeEach public void setUp() { sampleService = mock(SampleService.class); @@ -73,6 +78,9 @@ public void setUp() { analysisService = mock(AnalysisService.class); sequencingRun = mock(SequencingRun.class); + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + controller = new RESTSampleSequenceFilesController(sampleService, miseqRunService, sequencingObjectService, analysisService); From 03c3b7ae116dba91a84b32e69a227a96da74af96 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 21 Apr 2022 18:09:47 -0500 Subject: [PATCH 240/655] Moved changeset for this branch into a RENAME_WHEN_READY directory --- .../corefacility/bioinformatics/irida/database/all-changes.xml | 2 +- .../add-storage-type-to-file-tables.xml | 0 .../changesets/{22.05 => RENAME_WHEN_READY}/all-changes.xml | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/{22.05 => RENAME_WHEN_READY}/add-storage-type-to-file-tables.xml (100%) rename src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/{22.05 => RENAME_WHEN_READY}/all-changes.xml (100%) diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml index f3e70a39063..f4c0fc35161 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml @@ -70,5 +70,5 @@ - + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-file-tables.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/RENAME_WHEN_READY/add-storage-type-to-file-tables.xml similarity index 100% rename from src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/add-storage-type-to-file-tables.xml rename to src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/RENAME_WHEN_READY/add-storage-type-to-file-tables.xml diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/RENAME_WHEN_READY/all-changes.xml similarity index 100% rename from src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.05/all-changes.xml rename to src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/RENAME_WHEN_READY/all-changes.xml From 73d5b6c3461a3ea239dbafe8f027b160ef3cb558 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Sun, 1 May 2022 20:04:20 -0500 Subject: [PATCH 241/655] Fixed null pointer --- .../irida/ria/unit/utilities/FileUtilitiesTest.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java index f6b1c7f92fb..da43d086ead 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/utilities/FileUtilitiesTest.java @@ -7,12 +7,22 @@ import java.nio.file.Path; import java.nio.file.Paths; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageLocalUtilityImpl; +import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility; import ca.corefacility.bioinformatics.irida.ria.utilities.FileUtilities; +import ca.corefacility.bioinformatics.irida.util.IridaFiles; public class FileUtilitiesTest { - + private IridaFileStorageUtility iridaFileStorageUtility; + + @BeforeEach + public void setup () { + iridaFileStorageUtility = new IridaFileStorageLocalUtilityImpl(); + IridaFiles.setIridaFileStorageUtility(iridaFileStorageUtility); + } @Test public void testIsZippedFile() throws IOException { Path snpTreePath = Paths.get("src/test/resources/files/snp_tree.tree"); From d099f3d50adef9db4eb534dbd1f3b2c4c0b78009 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 26 Jul 2022 13:58:32 -0500 Subject: [PATCH 242/655] Fixed translation --- src/main/resources/i18n/messages.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index 661926cb74c..b21a9da6de6 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2599,7 +2599,7 @@ FastQC.minLength=Min. Length FastQC.maxLength=Max. Length FastQC.gcContent=GC Content FastQC.fetchingDetails=Fetching FastQC details -FastQC.fileDetails=File DetailsProjectReferenceFileController.java +FastQC.fileDetails=File Details FastQC.sequenceDetails=Sequence Details FastQC.noResults=A fastQC analysis has not been performed on this file. FastQC.fetchingData=Fetching fastqc data From 22d8ed4e3806c89fcfe76d81020e7ce20630d7d0 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 28 Jul 2022 14:53:37 -0500 Subject: [PATCH 243/655] Updated package to latest version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 256eb4d618a..65d2a1c9004 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -181,7 +181,7 @@ dependencies { } // Microsoft Azure - implementation("com.azure:azure-storage-blob:12.13.0") { + implementation("com.azure:azure-storage-blob:12.18.0") { exclude(group = "jakarta.xml.bind", module = "jakarta.xml.bind-api") exclude(group = "jakarta.activation", module = "jakarta.activation-api") } From 9331f47b2b0c6ca3caf050016af3c75997f8446b Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 31 Aug 2022 10:51:21 -0500 Subject: [PATCH 244/655] reading file with sheetjs --- .../SampleMetadataImportUploadFile.jsx | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx index 5ed47fee67e..b109bad578f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx @@ -4,12 +4,21 @@ import { useNavigate, useParams } from "react-router-dom"; import { setHeaders } from "../services/importReducer"; import { notification, Typography } from "antd"; import { DragUpload } from "../../../../components/files/DragUpload"; -import { setBaseUrl } from "../../../../utilities/url-utilities"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { useClearProjectSampleMetadataMutation } from "../../../../apis/metadata/metadata-import"; +import * as XLSX from "xlsx"; const { Text } = Typography; +function processFile(data) { + const workbook = XLSX.read(data, { type: "binary", raw: true }); + const firstSheet = workbook.SheetNames[0]; + const rows = XLSX.utils.sheet_to_row_object_array( + workbook.Sheets[firstSheet] + ); + console.log(rows); +} + /** * React component that displays Step #1 of the Sample Metadata Uploader. * This page is where the user selects the file to be uploaded. @@ -31,11 +40,21 @@ export function SampleMetadataImportUploadFile() { multiple: false, showUploadList: false, accept: [".xls", ".xlsx", ".csv"], - action: setBaseUrl( - `/ajax/projects/sample-metadata/upload/file?projectId=${projectId}` - ), + // action: setBaseUrl( + // `/ajax/projects/sample-metadata/upload/file?projectId=${projectId}` + // ), onChange(info) { const { status } = info.file; + if (info.file.status !== "uploading") { + let reader = new FileReader(); + if (reader.readAsBinaryString) { + reader.onload = (e) => { + processFile(reader.result); + }; + reader.readAsBinaryString(info.file.originFileObj); + } + return false; + } if (status === "done") { notification.success({ message: i18n( From 74c13698534ca6ca10433d13295c5483d17b7f9f Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 1 Sep 2022 14:13:45 -0500 Subject: [PATCH 245/655] persisting file contents --- .../SampleMetadataImportMapHeaders.jsx | 6 +- .../SampleMetadataImportUploadFile.jsx | 36 ++++++------ .../services/importReducer.js | 58 ++++++++++++++----- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx index c5dba445c76..ea89c08467e 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx @@ -26,8 +26,10 @@ export function SampleMetadataImportMapHeaders() { const [updateColumn] = useSetColumnProjectSampleMetadataMutation(); React.useEffect(() => { - setColumn(sampleNameColumn ? sampleNameColumn : headers[0]); - }, []); + if (!column) { + setColumn(sampleNameColumn ? sampleNameColumn : headers[0]); + } + }, [sampleNameColumn, headers]); const onSubmit = () => { updateColumn({ projectId, sampleNameColumn: column }) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx index b109bad578f..56d4ddb9b10 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx @@ -1,7 +1,11 @@ import React from "react"; import { useDispatch } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; -import { setHeaders } from "../services/importReducer"; +import { + setHeaders, + setMetadata, + setSampleNameColumn, +} from "../services/importReducer"; import { notification, Typography } from "antd"; import { DragUpload } from "../../../../components/files/DragUpload"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; @@ -10,15 +14,6 @@ import * as XLSX from "xlsx"; const { Text } = Typography; -function processFile(data) { - const workbook = XLSX.read(data, { type: "binary", raw: true }); - const firstSheet = workbook.SheetNames[0]; - const rows = XLSX.utils.sheet_to_row_object_array( - workbook.Sheets[firstSheet] - ); - console.log(rows); -} - /** * React component that displays Step #1 of the Sample Metadata Uploader. * This page is where the user selects the file to be uploaded. @@ -49,10 +44,21 @@ export function SampleMetadataImportUploadFile() { let reader = new FileReader(); if (reader.readAsBinaryString) { reader.onload = (e) => { - processFile(reader.result); + const workbook = XLSX.read(reader.result, { + type: "binary", + raw: true, + }); + const firstSheet = workbook.SheetNames[0]; + const rows = XLSX.utils.sheet_to_row_object_array( + workbook.Sheets[firstSheet] + ); + // dispatch(setSampleNameColumn(Object.keys(rows[0])[0])); + dispatch(setHeaders(Object.keys(rows[0]))); + dispatch(setMetadata(rows)); }; reader.readAsBinaryString(info.file.originFileObj); } + navigate(`/${projectId}/sample-metadata/upload/headers`); return false; } if (status === "done") { @@ -62,12 +68,8 @@ export function SampleMetadataImportUploadFile() { info.file.name ), }); - dispatch( - setHeaders( - info.file.response.headers, - info.file.response.sampleNameColumn - ) - ); + dispatch(setSampleNameColumn(info.file.response.sampleNameColumn)); + dispatch(setHeaders(info.file.response.headers)); navigate(`/${projectId}/sample-metadata/upload/headers`); } else if (status === "error") { setStatus("error"); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index adf67a06ca9..a6ba6bc8990 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -1,15 +1,41 @@ -import { createReducer, createAction } from "@reduxjs/toolkit"; +import { createAction, createReducer } from "@reduxjs/toolkit"; -const initialState = {} +const initialState = { + sampleNameColumn: "", + headers: [], + metadata: [], +}; /* -Redux action for project metadata. +Redux action for setting the sample name column. +For more information on redux actions see: https://redux-toolkit.js.org/api/createAction + */ +export const setSampleNameColumn = createAction( + `importReducer/setSampleNameColumn`, + (sampleNameColumn) => ({ + payload: { sampleNameColumn }, + }) +); + +/* +Redux action for setting the metadata headers. For more information on redux actions see: https://redux-toolkit.js.org/api/createAction */ export const setHeaders = createAction( - `rootReducers/setHeaders`, - (headers, sampleNameColumn) => ({ - payload: { headers, sampleNameColumn } + `importReducer/setHeaders`, + (headers) => ({ + payload: { headers }, + }) +); + +/* +Redux action for setting the project metadata. +For more information on redux actions see: https://redux-toolkit.js.org/api/createAction + */ +export const setMetadata = createAction( + `importReducer/setMetadata`, + (metadata) => ({ + payload: { metadata }, }) ); @@ -17,12 +43,14 @@ export const setHeaders = createAction( Redux reducer for project metadata. For more information on redux reducers see: https://redux-toolkit.js.org/api/createReducer */ -export const importReducer = createReducer( - initialState, - (builder) => { - builder.addCase(setHeaders, (state, action) => { - state.headers = action.payload.headers; - state.sampleNameColumn = action.payload.sampleNameColumn; - }); - } -); \ No newline at end of file +export const importReducer = createReducer(initialState, (builder) => { + builder.addCase(setSampleNameColumn, (state, action) => { + state.sampleNameColumn = action.payload.sampleNameColumn; + }); + builder.addCase(setHeaders, (state, action) => { + state.headers = action.payload.headers; + }); + builder.addCase(setMetadata, (state, action) => { + state.metadata = action.payload.metadata; + }); +}); From ece38b99a52f55ade3dda6dbc16a3b86e0f4e935 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 9 Sep 2022 12:34:30 -0500 Subject: [PATCH 246/655] setting sample name column --- .../components/SampleMetadataImportMapHeaders.jsx | 13 +++++-------- .../components/SampleMetadataImportUploadFile.jsx | 10 ---------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx index ea89c08467e..37ee8f5682a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx @@ -1,14 +1,14 @@ import React from "react"; -import { useSelector } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; import { Button, Radio, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { BlockRadioInput } from "../../../../components/ant.design/forms/BlockRadioInput"; -import { useSetColumnProjectSampleMetadataMutation } from "../../../../apis/metadata/metadata-import"; import { IconArrowLeft, IconArrowRight, } from "../../../../components/icons/Icons"; +import { setSampleNameColumn } from "../services/importReducer"; const { Text } = Typography; @@ -23,7 +23,7 @@ export function SampleMetadataImportMapHeaders() { const navigate = useNavigate(); const [column, setColumn] = React.useState(); const { headers, sampleNameColumn } = useSelector((state) => state.reducer); - const [updateColumn] = useSetColumnProjectSampleMetadataMutation(); + const dispatch = useDispatch(); React.useEffect(() => { if (!column) { @@ -32,11 +32,8 @@ export function SampleMetadataImportMapHeaders() { }, [sampleNameColumn, headers]); const onSubmit = () => { - updateColumn({ projectId, sampleNameColumn: column }) - .unwrap() - .then((payload) => { - navigate(`/${projectId}/sample-metadata/upload/review`); - }); + dispatch(setSampleNameColumn(column)); + navigate(`/${projectId}/sample-metadata/upload/review`); }; return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx index 56d4ddb9b10..967549f7498 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx @@ -9,7 +9,6 @@ import { import { notification, Typography } from "antd"; import { DragUpload } from "../../../../components/files/DragUpload"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; -import { useClearProjectSampleMetadataMutation } from "../../../../apis/metadata/metadata-import"; import * as XLSX from "xlsx"; const { Text } = Typography; @@ -25,19 +24,11 @@ export function SampleMetadataImportUploadFile() { const navigate = useNavigate(); const dispatch = useDispatch(); const [status, setStatus] = React.useState("process"); - const [clearStorage] = useClearProjectSampleMetadataMutation(); - - React.useEffect(() => { - clearStorage(projectId); - }, [clearStorage, projectId]); const options = { multiple: false, showUploadList: false, accept: [".xls", ".xlsx", ".csv"], - // action: setBaseUrl( - // `/ajax/projects/sample-metadata/upload/file?projectId=${projectId}` - // ), onChange(info) { const { status } = info.file; if (info.file.status !== "uploading") { @@ -52,7 +43,6 @@ export function SampleMetadataImportUploadFile() { const rows = XLSX.utils.sheet_to_row_object_array( workbook.Sheets[firstSheet] ); - // dispatch(setSampleNameColumn(Object.keys(rows[0])[0])); dispatch(setHeaders(Object.keys(rows[0]))); dispatch(setMetadata(rows)); }; From b60996bd2bb31ac1a8f486aa5d5f87656aac89b6 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 9 Sep 2022 14:17:45 -0500 Subject: [PATCH 247/655] starting to show metadata in table on review step --- .../components/SampleMetadataImportReview.jsx | 127 ++++++++---------- 1 file changed, 58 insertions(+), 69 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 5a32dad168c..d2e984aa454 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -2,10 +2,7 @@ import React from "react"; import { useNavigate, useParams } from "react-router-dom"; import { Alert, Button, Table, Tag, Tooltip, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; -import { - useGetProjectSampleMetadataQuery, - useSaveProjectSampleMetadataMutation, -} from "../../../../apis/metadata/metadata-import"; +import { useSaveProjectSampleMetadataMutation } from "../../../../apis/metadata/metadata-import"; import { IconArrowLeft, IconArrowRight, @@ -13,6 +10,7 @@ import { } from "../../../../components/icons/Icons"; import { red1, red2, red5 } from "../../../../styles/colors"; import styled from "styled-components"; +import { useSelector } from "react-redux"; const { Paragraph, Text } = Typography; @@ -49,8 +47,8 @@ export function SampleMetadataImportReview() { const [columns, setColumns] = React.useState([]); const [selected, setSelected] = React.useState([]); const [valid, setValid] = React.useState(true); - const { data = {}, isFetching, isSuccess } = useGetProjectSampleMetadataQuery( - projectId + const { headers, sampleNameColumn, metadata } = useSelector( + (state) => state.reducer ); const [saveMetadata] = useSaveProjectSampleMetadataMutation(); @@ -97,76 +95,68 @@ export function SampleMetadataImportReview() { }; React.useEffect(() => { - if (isSuccess) { - setValid(!data.rows.some((row) => row.isSampleNameValid === false)); - - const index = data.headers.findIndex( - (item) => item === data.sampleNameColumn - ); - - const headers = [...data.headers]; - - const sample = headers.splice(index, 1)[0]; - - const sampleColumn = { - title: sample, - dataIndex: sample, - fixed: "left", - width: 100, - render(text, item) { - return { - props: { - style: { background: item.isSampleNameValid ? null : red1 }, - }, - children: item.entry[sample], - }; - }, - }; - - const savedColumn = { - dataIndex: "saved", - fixed: "left", - width: 10, - render: (text, item) => { - if (item.saved === false) - return ( - - - - ); - }, - }; + console.log(metadata); + // setValid(!data.rows.some((row) => row.isSampleNameValid === false)); + + const sampleColumn = { + title: sampleNameColumn, + dataIndex: sampleNameColumn, + fixed: "left", + width: 100, + render(text, item) { + return { + props: { + style: { background: item.isSampleNameValid ? null : red1 }, + }, + children: item[sampleNameColumn], + }; + }, + }; + + const savedColumn = { + dataIndex: "saved", + fixed: "left", + width: 10, + render: (text, item) => { + if (item.saved === false) + return ( + + + + ); + }, + }; - const otherColumns = headers.map((header) => ({ + const otherColumns = headers + .filter((header) => header !== sampleNameColumn) + .map((header) => ({ title: header, dataIndex: header, - render: (text, item) => item.entry[header], + // render: (text, item) => item[header], })); - const updatedColumns = [ - savedColumn, - sampleColumn, - tagColumn, - ...otherColumns, - ]; + console.log(otherColumns); + + const updatedColumns = [ + savedColumn, + sampleColumn, + tagColumn, + ...otherColumns, + ]; - setColumns(updatedColumns); - setSelected( - data.rows.map((row) => { - if ( - row.isSampleNameValid && - (row.saved === null || row.saved === true) - ) - return row.rowKey; - }) - ); - } - }, [data, isSuccess]); + setColumns(updatedColumns); + // setSelected( + // metadata.map((row) => { + // if (row.isSampleNameValid && (row.saved === null || row.saved === true)) + // return row.rowKey; + // }) + // ); + }, []); const save = () => { - const sampleNames = data.rows + const sampleNames = metadata .filter((row) => selected.includes(row.rowKey)) - .map((row) => row.entry[data.sampleNameColumn]); + .map((row) => row.entry[sampleNameColumn]); saveMetadata({ projectId, sampleNames }) .unwrap() .then((payload) => { @@ -199,13 +189,12 @@ export function SampleMetadataImportReview() { row.rowKey} - loading={isFetching} rowClassName={(record, index) => record.saved === false ? "row-error" : null } rowSelection={rowSelection} columns={columns} - dataSource={data.rows} + dataSource={metadata} scroll={{ x: "max-content", y: 600 }} pagination={false} /> From becabf5981332590e8fad59fe15b44a08fbae0fa Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 9 Sep 2022 15:46:12 -0500 Subject: [PATCH 248/655] starting to show metadata in table on review step --- .../components/SampleMetadataImportReview.jsx | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index d2e984aa454..822695c9430 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -96,14 +96,14 @@ export function SampleMetadataImportReview() { React.useEffect(() => { console.log(metadata); - // setValid(!data.rows.some((row) => row.isSampleNameValid === false)); + setValid(!metadata.some((row) => row.isSampleNameValid === false)); const sampleColumn = { title: sampleNameColumn, dataIndex: sampleNameColumn, fixed: "left", width: 100, - render(text, item) { + onCell(text, item) { return { props: { style: { background: item.isSampleNameValid ? null : red1 }, @@ -132,11 +132,8 @@ export function SampleMetadataImportReview() { .map((header) => ({ title: header, dataIndex: header, - // render: (text, item) => item[header], })); - console.log(otherColumns); - const updatedColumns = [ savedColumn, sampleColumn, @@ -145,12 +142,12 @@ export function SampleMetadataImportReview() { ]; setColumns(updatedColumns); - // setSelected( - // metadata.map((row) => { - // if (row.isSampleNameValid && (row.saved === null || row.saved === true)) - // return row.rowKey; - // }) - // ); + setSelected( + metadata.map((row) => { + if (row.isSampleNameValid && (row.saved === null || row.saved === true)) + return row.rowKey; + }) + ); }, []); const save = () => { @@ -188,10 +185,8 @@ export function SampleMetadataImportReview() { )} row.rowKey} - rowClassName={(record, index) => - record.saved === false ? "row-error" : null - } + rowKey={(row, index) => `metadata-uploader-row-${index}`} + rowClassName={(record) => (record.saved === false ? "row-error" : null)} rowSelection={rowSelection} columns={columns} dataSource={metadata} From 9789f438ffde7d80cebf76c2fa55a9daabe697ad Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Mon, 12 Sep 2022 15:54:25 -0500 Subject: [PATCH 249/655] supplementing metadata after sample name column is selected --- .../components/SampleMetadataImportReview.jsx | 2 +- .../samples-metadata-import/services/importReducer.js | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 822695c9430..774d325109e 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -185,7 +185,7 @@ export function SampleMetadataImportReview() { )} `metadata-uploader-row-${index}`} + rowKey={(row) => row.rowKey} rowClassName={(record) => (record.saved === false ? "row-error" : null)} rowSelection={rowSelection} columns={columns} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index a6ba6bc8990..58de374cf14 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -1,4 +1,5 @@ import { createAction, createReducer } from "@reduxjs/toolkit"; +import { validateSampleName } from "../../../../apis/metadata/sample-utils"; const initialState = { sampleNameColumn: "", @@ -46,6 +47,14 @@ For more information on redux reducers see: https://redux-toolkit.js.org/api/cre export const importReducer = createReducer(initialState, (builder) => { builder.addCase(setSampleNameColumn, (state, action) => { state.sampleNameColumn = action.payload.sampleNameColumn; + state.metadata = state.metadata.map((item, index) => { + return { + ...item, + rowKey: `metadata-uploader-row-${index}`, + isSampleNameValid: validateSampleName(item[state.sampleNameColumn]), + saved: null, + }; + }); }); builder.addCase(setHeaders, (state, action) => { state.headers = action.payload.headers; From 5876171bc422f779850b50653c37f6a15478414b Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 13 Sep 2022 09:19:51 -0500 Subject: [PATCH 250/655] fixing onCell for sample name collumn --- .../components/SampleMetadataImportReview.jsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 774d325109e..a6ffb6faec5 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -103,12 +103,11 @@ export function SampleMetadataImportReview() { dataIndex: sampleNameColumn, fixed: "left", width: 100, - onCell(text, item) { + onCell: (item) => { return { - props: { - style: { background: item.isSampleNameValid ? null : red1 }, + style: { + background: item.isSampleNameValid === true ? null : red1, }, - children: item[sampleNameColumn], }; }, }; From f0aebf6b89f3d3b3c7e97367661379069a00a0be Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 14 Sep 2022 16:36:15 -0500 Subject: [PATCH 251/655] working on validating sample names --- .../resources/js/apis/projects/samples.ts | 5 ++ .../SampleMetadataImportMapHeaders.jsx | 2 +- .../components/SampleMetadataImportReview.jsx | 3 + .../SampleMetadataImportUploadFile.jsx | 12 +--- .../services/importReducer.js | 58 ++++++++++++------- 5 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index 12792fcf2c5..d1887c445ca 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -5,6 +5,7 @@ import { } from "../../types/irida"; import { getProjectIdFromUrl, setBaseUrl } from "../../utilities/url-utilities"; import { get, post } from "../requests"; +import axios from "axios"; export interface SequencingFiles { singles: SingleEndSequenceFile[]; @@ -70,6 +71,10 @@ export const { useShareSamplesWithProjectMutation, } = samplesApi; +export async function validateSamples({ projectId, body }) { + return await axios.post(`${URL}/${projectId}/samples/validate`, body); +} + /** * Server side validation of a new sample name. * @param name - sample name to validate diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx index 37ee8f5682a..8a0aceaf1d7 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx @@ -32,7 +32,7 @@ export function SampleMetadataImportMapHeaders() { }, [sampleNameColumn, headers]); const onSubmit = () => { - dispatch(setSampleNameColumn(column)); + dispatch(setSampleNameColumn({ projectId, column })); navigate(`/${projectId}/sample-metadata/upload/review`); }; diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index a6ffb6faec5..051dec6aeb9 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -95,7 +95,10 @@ export function SampleMetadataImportReview() { }; React.useEffect(() => { + console.log(headers); + console.log(sampleNameColumn); console.log(metadata); + setValid(!metadata.some((row) => row.isSampleNameValid === false)); const sampleColumn = { diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx index 967549f7498..07df20bf84f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx @@ -1,11 +1,7 @@ import React from "react"; import { useDispatch } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; -import { - setHeaders, - setMetadata, - setSampleNameColumn, -} from "../services/importReducer"; +import { setHeaders, setMetadata } from "../services/importReducer"; import { notification, Typography } from "antd"; import { DragUpload } from "../../../../components/files/DragUpload"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; @@ -31,7 +27,7 @@ export function SampleMetadataImportUploadFile() { accept: [".xls", ".xlsx", ".csv"], onChange(info) { const { status } = info.file; - if (info.file.status !== "uploading") { + if (status !== "uploading") { let reader = new FileReader(); if (reader.readAsBinaryString) { reader.onload = (e) => { @@ -48,8 +44,6 @@ export function SampleMetadataImportUploadFile() { }; reader.readAsBinaryString(info.file.originFileObj); } - navigate(`/${projectId}/sample-metadata/upload/headers`); - return false; } if (status === "done") { notification.success({ @@ -58,8 +52,6 @@ export function SampleMetadataImportUploadFile() { info.file.name ), }); - dispatch(setSampleNameColumn(info.file.response.sampleNameColumn)); - dispatch(setHeaders(info.file.response.headers)); navigate(`/${projectId}/sample-metadata/upload/headers`); } else if (status === "error") { setStatus("error"); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 58de374cf14..6ce086fd357 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -1,5 +1,10 @@ -import { createAction, createReducer } from "@reduxjs/toolkit"; +import { + createAction, + createAsyncThunk, + createReducer, +} from "@reduxjs/toolkit"; import { validateSampleName } from "../../../../apis/metadata/sample-utils"; +import { validateSamples } from "../../../../apis/projects/samples"; const initialState = { sampleNameColumn: "", @@ -7,15 +12,34 @@ const initialState = { metadata: [], }; -/* -Redux action for setting the sample name column. -For more information on redux actions see: https://redux-toolkit.js.org/api/createAction - */ -export const setSampleNameColumn = createAction( +export const setSampleNameColumn = createAsyncThunk( `importReducer/setSampleNameColumn`, - (sampleNameColumn) => ({ - payload: { sampleNameColumn }, - }) + async ({ projectId, column }, { getState }) => { + const state = getState(); + const metadata = state.reducer.metadata; + const response = await validateSamples({ + projectId: projectId, + body: { + samples: metadata.map((row) => ({ + name: row[column], + })), + }, + }); + + const updatedMetadata = metadata.map((metadataItem, index) => { + return { + ...metadataItem, + rowKey: `metadata-uploader-row-${index}`, + isSampleNameValid: validateSampleName(metadataItem[column]), + foundSampleId: response.data.samples + .find((sample) => metadataItem[column] === sample.name) + .ids?.at(0), + saved: null, + }; + }); + + return { sampleNameColumn: column, metadata: updatedMetadata }; + } ); /* @@ -45,21 +69,15 @@ Redux reducer for project metadata. For more information on redux reducers see: https://redux-toolkit.js.org/api/createReducer */ export const importReducer = createReducer(initialState, (builder) => { - builder.addCase(setSampleNameColumn, (state, action) => { - state.sampleNameColumn = action.payload.sampleNameColumn; - state.metadata = state.metadata.map((item, index) => { - return { - ...item, - rowKey: `metadata-uploader-row-${index}`, - isSampleNameValid: validateSampleName(item[state.sampleNameColumn]), - saved: null, - }; - }); - }); builder.addCase(setHeaders, (state, action) => { state.headers = action.payload.headers; }); builder.addCase(setMetadata, (state, action) => { state.metadata = action.payload.metadata; }); + builder.addCase(setSampleNameColumn.fulfilled, (state, action) => { + console.log(action); + state.sampleNameColumn = action.payload.sampleNameColumn; + state.metadata = action.payload.metadata; + }); }); From 86daff75851f3cbe0b74afbd7a29bb87335daabb Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 15 Sep 2022 11:28:07 -0500 Subject: [PATCH 252/655] navigating to review page once setting sample name is complete --- .../components/SampleMetadataImportMapHeaders.jsx | 8 +++++--- .../components/SampleMetadataImportReview.jsx | 10 +++++----- .../samples-metadata-import/services/importReducer.js | 10 ++++++++-- .../js/pages/projects/samples-metadata-import/store.js | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx index 8a0aceaf1d7..8b0fcf4cd46 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx @@ -22,7 +22,9 @@ export function SampleMetadataImportMapHeaders() { const { projectId } = useParams(); const navigate = useNavigate(); const [column, setColumn] = React.useState(); - const { headers, sampleNameColumn } = useSelector((state) => state.reducer); + const { headers, sampleNameColumn } = useSelector( + (state) => state.importReducer + ); const dispatch = useDispatch(); React.useEffect(() => { @@ -31,8 +33,8 @@ export function SampleMetadataImportMapHeaders() { } }, [sampleNameColumn, headers]); - const onSubmit = () => { - dispatch(setSampleNameColumn({ projectId, column })); + const onSubmit = async () => { + await dispatch(setSampleNameColumn({ projectId, column })); navigate(`/${projectId}/sample-metadata/upload/review`); }; diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 051dec6aeb9..d7134533eca 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -48,10 +48,14 @@ export function SampleMetadataImportReview() { const [selected, setSelected] = React.useState([]); const [valid, setValid] = React.useState(true); const { headers, sampleNameColumn, metadata } = useSelector( - (state) => state.reducer + (state) => state.importReducer ); const [saveMetadata] = useSaveProjectSampleMetadataMutation(); + console.log("headers", headers); + console.log("sampleNameColumn", sampleNameColumn); + console.log("metadata", metadata); + const tagColumn = { title: "", dataIndex: "tags", @@ -95,10 +99,6 @@ export function SampleMetadataImportReview() { }; React.useEffect(() => { - console.log(headers); - console.log(sampleNameColumn); - console.log(metadata); - setValid(!metadata.some((row) => row.isSampleNameValid === false)); const sampleColumn = { diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 6ce086fd357..5ef01064b10 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -2,6 +2,7 @@ import { createAction, createAsyncThunk, createReducer, + current, } from "@reduxjs/toolkit"; import { validateSampleName } from "../../../../apis/metadata/sample-utils"; import { validateSamples } from "../../../../apis/projects/samples"; @@ -12,11 +13,15 @@ const initialState = { metadata: [], }; +/* +Redux sync thunk for setting the sample name column and enriching the metadata. +For more information on redux async thunks see: https://redux-toolkit.js.org/api/createAsyncThunk +*/ export const setSampleNameColumn = createAsyncThunk( `importReducer/setSampleNameColumn`, async ({ projectId, column }, { getState }) => { const state = getState(); - const metadata = state.reducer.metadata; + const metadata = state.importReducer.metadata; const response = await validateSamples({ projectId: projectId, body: { @@ -76,8 +81,9 @@ export const importReducer = createReducer(initialState, (builder) => { state.metadata = action.payload.metadata; }); builder.addCase(setSampleNameColumn.fulfilled, (state, action) => { - console.log(action); + console.log("before", current(state)); state.sampleNameColumn = action.payload.sampleNameColumn; state.metadata = action.payload.metadata; + console.log("after", current(state)); }); }); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js index 19267b16241..8cf56190fdd 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js @@ -8,7 +8,7 @@ For more information on redux stores see: https://redux.js.org/tutorials/fundame */ export default configureStore({ reducer: { - reducer: importReducer, + importReducer, [metadataImportApi.reducerPath]: metadataImportApi.reducer, }, middleware: (getDefaultMiddleware) => From 1f2f2b0045e7c631057fd6544bf3feaa53e86733 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 16 Sep 2022 15:33:46 -0500 Subject: [PATCH 253/655] starting to create/update samples with metadata --- .../js/apis/metadata/metadata-import.js | 80 ------------------ .../resources/js/apis/projects/samples.ts | 11 +++ .../components/SampleMetadataImportReview.jsx | 32 ++++--- .../services/importReducer.js | 83 +++++++++++++++++-- .../projects/samples-metadata-import/store.js | 5 +- 5 files changed, 105 insertions(+), 106 deletions(-) delete mode 100644 src/main/webapp/resources/js/apis/metadata/metadata-import.js diff --git a/src/main/webapp/resources/js/apis/metadata/metadata-import.js b/src/main/webapp/resources/js/apis/metadata/metadata-import.js deleted file mode 100644 index ecce5537aa9..00000000000 --- a/src/main/webapp/resources/js/apis/metadata/metadata-import.js +++ /dev/null @@ -1,80 +0,0 @@ -import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; -import { setBaseUrl } from "../../utilities/url-utilities"; -import { validateSampleName } from "./sample-utils"; - -const BASE_URL = setBaseUrl(`ajax/projects/sample-metadata/upload`); - -/** - * Redux API for Sample Metadata - */ -export const metadataImportApi = createApi({ - reducerPath: `metadataImportApi`, - baseQuery: fetchBaseQuery({ baseUrl: BASE_URL }), - tagTypes: ["MetadataImport"], - endpoints: (build) => ({ - getProjectSampleMetadata: build.query({ - query: (projectId) => ({ - url: `/getMetadata`, - params: { - projectId, - }, - }), - /** - Transforming the response to include if the row has a valid sample name and a unique row key for rendering the ant design table. - */ - transformResponse(response) { - const transformed = { - ...response, - rows: response.rows.map((row, index) => ({ - ...row, - rowKey: `row-${index}`, - isSampleNameValid: validateSampleName( - row.entry[response.sampleNameColumn] - ), - })), - }; - return transformed; - }, - providesTags: ["MetadataImport"], - }), - clearProjectSampleMetadata: build.mutation({ - query: (projectId) => ({ - url: `/clear`, - method: "DELETE", - params: { - projectId, - }, - }), - invalidatesTags: ["MetadataImport"], - }), - setColumnProjectSampleMetadata: build.mutation({ - query: ({ projectId, sampleNameColumn }) => ({ - url: `/setSampleColumn`, - method: "PUT", - params: { - projectId, - sampleNameColumn, - }, - }), - invalidatesTags: ["MetadataImport"], - }), - saveProjectSampleMetadata: build.mutation({ - query: ({ projectId, sampleNames }) => ({ - url: `/save`, - method: "POST", - params: { - projectId, - sampleNames, - }, - }), - invalidatesTags: ["MetadataImport"], - }), - }), -}); - -export const { - useGetProjectSampleMetadataQuery, - useClearProjectSampleMetadataMutation, - useSetColumnProjectSampleMetadataMutation, - useSaveProjectSampleMetadataMutation, -} = metadataImportApi; diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index d1887c445ca..a970a5ebb32 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -75,6 +75,17 @@ export async function validateSamples({ projectId, body }) { return await axios.post(`${URL}/${projectId}/samples/validate`, body); } +export async function createSample({ projectId, body }) { + return await axios.post(`${URL}/${projectId}/samples/add-sample`, body); +} + +export async function updateSample({ projectId, sampleId, body }) { + return await axios.patch( + `${URL}/${projectId}/samples/add-sample/${sampleId}`, + body + ); +} + /** * Server side validation of a new sample name. * @param name - sample name to validate diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index d7134533eca..d222fd57f03 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -2,7 +2,6 @@ import React from "react"; import { useNavigate, useParams } from "react-router-dom"; import { Alert, Button, Table, Tag, Tooltip, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; -import { useSaveProjectSampleMetadataMutation } from "../../../../apis/metadata/metadata-import"; import { IconArrowLeft, IconArrowRight, @@ -10,7 +9,8 @@ import { } from "../../../../components/icons/Icons"; import { red1, red2, red5 } from "../../../../styles/colors"; import styled from "styled-components"; -import { useSelector } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; +import { saveMetadata } from "../services/importReducer"; const { Paragraph, Text } = Typography; @@ -50,11 +50,7 @@ export function SampleMetadataImportReview() { const { headers, sampleNameColumn, metadata } = useSelector( (state) => state.importReducer ); - const [saveMetadata] = useSaveProjectSampleMetadataMutation(); - - console.log("headers", headers); - console.log("sampleNameColumn", sampleNameColumn); - console.log("metadata", metadata); + const dispatch = useDispatch(); const tagColumn = { title: "", @@ -153,16 +149,18 @@ export function SampleMetadataImportReview() { }, []); const save = () => { - const sampleNames = metadata - .filter((row) => selected.includes(row.rowKey)) - .map((row) => row.entry[sampleNameColumn]); - saveMetadata({ projectId, sampleNames }) - .unwrap() - .then((payload) => { - navigate(`/${projectId}/sample-metadata/upload/complete`, { - state: { statusMessage: payload.message }, - }); - }); + const selectedMetadataKeys = metadata + .filter((metadataItem) => selected.includes(metadataItem.rowKey)) + .map((metadataItem) => metadataItem.rowKey); + + dispatch(saveMetadata({ projectId, selectedMetadataKeys })); + // saveMetadata({ projectId, sampleNames }) + // .unwrap() + // .then((payload) => { + // navigate(`/${projectId}/sample-metadata/upload/complete`, { + // state: { statusMessage: payload.message }, + // }); + // }); }; return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 5ef01064b10..3e42e1af181 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -2,10 +2,13 @@ import { createAction, createAsyncThunk, createReducer, - current, } from "@reduxjs/toolkit"; import { validateSampleName } from "../../../../apis/metadata/sample-utils"; -import { validateSamples } from "../../../../apis/projects/samples"; +import { + createSample, + updateSample, + validateSamples, +} from "../../../../apis/projects/samples"; const initialState = { sampleNameColumn: "", @@ -14,7 +17,76 @@ const initialState = { }; /* -Redux sync thunk for setting the sample name column and enriching the metadata. +Redux async thunk for saving the metadata to samples. +For more information on redux async thunks see: https://redux-toolkit.js.org/api/createAsyncThunk + */ +export const saveMetadata = createAsyncThunk( + `importReducer/saveMetadata`, + async ({ projectId, selectedMetadataKeys }, { getState }) => { + const state = getState(); + const sampleNameColumn = state.importReducer.sampleNameColumn; + const headers = state.importReducer.headers; + const metadata = state.importReducer.metadata; + const updatedMetadata = [...metadata]; + + for (const [index, metadataItem] of updatedMetadata.entries()) { + if (selectedMetadataKeys.includes(metadataItem.rowKey)) { + const name = metadataItem[sampleNameColumn]; + const metadataFields = Object.entries(metadataItem) + .filter( + ([key, value]) => headers.includes(key) && key !== sampleNameColumn + ) + .map(([key, value]) => ({ field: key, value })); + const sampleId = metadataItem.foundSampleId; + + if (sampleId) { + await updateSample({ + projectId, + sampleId, + body: { + name, + // TODO: Don't overwrite organism & description + metadata: metadataFields, + }, + }) + .then((response) => { + console.log("UPDATE SAMPLE RESPONSE"); + console.log(response); + updatedMetadata.saved = true; + }) + .catch((error) => { + console.log("UPDATE SAMPLE ERROR"); + console.log(error); + metadata[index].saved = false; + }); + } else { + await createSample({ + projectId, + body: { + name, + metadata: metadataFields, + }, + }) + .then((response) => { + console.log("CREATE SAMPLE RESPONSE"); + console.log(response); + updatedMetadata.saved = true; + }) + .catch((error) => { + console.log("CREATE SAMPLE ERROR"); + console.log(error); + metadata[index].saved = false; + metadata[index].error = error; + }); + } + } + } + return { metadata: updatedMetadata }; + } +); + +/* +Redux async thunk for setting the sample name column and enriching the metadata. For more information on redux async thunks see: https://redux-toolkit.js.org/api/createAsyncThunk */ export const setSampleNameColumn = createAsyncThunk( @@ -81,9 +153,10 @@ export const importReducer = createReducer(initialState, (builder) => { state.metadata = action.payload.metadata; }); builder.addCase(setSampleNameColumn.fulfilled, (state, action) => { - console.log("before", current(state)); state.sampleNameColumn = action.payload.sampleNameColumn; state.metadata = action.payload.metadata; - console.log("after", current(state)); + }); + builder.addCase(saveMetadata.fulfilled, (state, action) => { + state.metadata = action.payload.metadata; }); }); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js index 8cf56190fdd..ff02574cfb6 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js @@ -1,5 +1,4 @@ import { configureStore } from "@reduxjs/toolkit"; -import { metadataImportApi } from "../../../apis/metadata/metadata-import"; import { importReducer } from "./services/importReducer"; /* @@ -9,8 +8,6 @@ For more information on redux stores see: https://redux.js.org/tutorials/fundame export default configureStore({ reducer: { importReducer, - [metadataImportApi.reducerPath]: metadataImportApi.reducer, }, - middleware: (getDefaultMiddleware) => - getDefaultMiddleware().concat(metadataImportApi.middleware), + middleware: (getDefaultMiddleware) => getDefaultMiddleware(), }); From f141727805f2c634dbbef978859ac08210c5f978 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 16 Sep 2022 16:24:37 -0500 Subject: [PATCH 254/655] displaying stats on complete page --- .../SampleMetadataImportComplete.jsx | 38 +++++++++++++++++-- .../components/SampleMetadataImportReview.jsx | 12 ++---- .../services/importReducer.js | 12 +++--- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx index ab929aee284..959584459f7 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx @@ -1,8 +1,9 @@ import React from "react"; -import { useParams, useLocation } from "react-router-dom"; +import { useParams } from "react-router-dom"; import { Button, Result } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { setBaseUrl } from "../../../../utilities/url-utilities"; +import { useSelector } from "react-redux"; /** * React component that displays Step #4 of the Sample Metadata Uploader. @@ -11,7 +12,38 @@ import { setBaseUrl } from "../../../../utilities/url-utilities"; * @constructor */ export function SampleMetadataImportComplete() { - const location = useLocation(); + const { metadata } = useSelector((state) => state.importReducer); + + const samplesUpdatedCount = metadata.filter( + (metadataItem) => + metadataItem.saved === true && metadataItem.foundSampleId !== null + ).length; + + const samplesCreatedCount = metadata.filter( + (metadataItem) => + metadataItem.saved === true && metadataItem.foundSampleId === null + ).length; + + let stats = + samplesUpdatedCount == 1 + ? i18n( + "server.metadataimport.results.save.success.single-updated", + samplesUpdatedCount + ) + : i18n( + "server.metadataimport.results.save.success.multiple-updated", + samplesUpdatedCount + ); + stats += + samplesCreatedCount == 1 + ? i18n( + "server.metadataimport.results.save.success.single-created", + samplesCreatedCount + ) + : i18n( + "server.metadataimport.results.save.success.multiple-created", + samplesCreatedCount + ); const { projectId } = useParams(); @@ -20,7 +52,7 @@ export function SampleMetadataImportComplete() { { + const save = async () => { const selectedMetadataKeys = metadata .filter((metadataItem) => selected.includes(metadataItem.rowKey)) .map((metadataItem) => metadataItem.rowKey); - dispatch(saveMetadata({ projectId, selectedMetadataKeys })); - // saveMetadata({ projectId, sampleNames }) - // .unwrap() - // .then((payload) => { - // navigate(`/${projectId}/sample-metadata/upload/complete`, { - // state: { statusMessage: payload.message }, - // }); - // }); + await dispatch(saveMetadata({ projectId, selectedMetadataKeys })); + navigate(`/${projectId}/sample-metadata/upload/complete`); }; return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 3e42e1af181..f1881f0aac7 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -27,7 +27,7 @@ export const saveMetadata = createAsyncThunk( const sampleNameColumn = state.importReducer.sampleNameColumn; const headers = state.importReducer.headers; const metadata = state.importReducer.metadata; - const updatedMetadata = [...metadata]; + const updatedMetadata = JSON.parse(JSON.stringify(metadata)); for (const [index, metadataItem] of updatedMetadata.entries()) { if (selectedMetadataKeys.includes(metadataItem.rowKey)) { @@ -52,12 +52,12 @@ export const saveMetadata = createAsyncThunk( .then((response) => { console.log("UPDATE SAMPLE RESPONSE"); console.log(response); - updatedMetadata.saved = true; + updatedMetadata[index].saved = true; }) .catch((error) => { console.log("UPDATE SAMPLE ERROR"); console.log(error); - metadata[index].saved = false; + updatedMetadata[index].saved = false; }); } else { await createSample({ @@ -70,13 +70,13 @@ export const saveMetadata = createAsyncThunk( .then((response) => { console.log("CREATE SAMPLE RESPONSE"); console.log(response); - updatedMetadata.saved = true; + updatedMetadata[index].saved = true; }) .catch((error) => { console.log("CREATE SAMPLE ERROR"); console.log(error); - metadata[index].saved = false; - metadata[index].error = error; + updatedMetadata[index].saved = false; + updatedMetadata[index].error = error; }); } } From b70cf425736a8e5b68fb3b6a443371bdfa26630a Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 20 Sep 2022 14:20:25 -0500 Subject: [PATCH 255/655] fixing stats --- .../components/SampleMetadataImportComplete.jsx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx index 959584459f7..6ae6ed2de2c 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx @@ -13,15 +13,12 @@ import { useSelector } from "react-redux"; */ export function SampleMetadataImportComplete() { const { metadata } = useSelector((state) => state.importReducer); - const samplesUpdatedCount = metadata.filter( - (metadataItem) => - metadataItem.saved === true && metadataItem.foundSampleId !== null + (metadataItem) => metadataItem.saved === true && metadataItem.foundSampleId ).length; const samplesCreatedCount = metadata.filter( - (metadataItem) => - metadataItem.saved === true && metadataItem.foundSampleId === null + (metadataItem) => metadataItem.saved === true && !metadataItem.foundSampleId ).length; let stats = From 5ab8c35bc170b3d371ce534f793af59e1bf33556 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 20 Sep 2022 16:58:12 -0500 Subject: [PATCH 256/655] showing errors --- .../components/SampleMetadataImportReview.jsx | 11 +++++++++-- .../samples-metadata-import/services/importReducer.js | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index ed0907b5a77..45afc920b71 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -153,8 +153,15 @@ export function SampleMetadataImportReview() { .filter((metadataItem) => selected.includes(metadataItem.rowKey)) .map((metadataItem) => metadataItem.rowKey); - await dispatch(saveMetadata({ projectId, selectedMetadataKeys })); - navigate(`/${projectId}/sample-metadata/upload/complete`); + const response = await dispatch( + saveMetadata({ projectId, selectedMetadataKeys }) + ); + + if ( + response.payload.metadata.every((metadataItem) => !metadataItem.error) + ) { + navigate(`/${projectId}/sample-metadata/upload/complete`); + } }; return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index f1881f0aac7..a2fb3a0c8dc 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -58,6 +58,7 @@ export const saveMetadata = createAsyncThunk( console.log("UPDATE SAMPLE ERROR"); console.log(error); updatedMetadata[index].saved = false; + updatedMetadata[index].error = error.response.data.error; }); } else { await createSample({ @@ -76,11 +77,12 @@ export const saveMetadata = createAsyncThunk( console.log("CREATE SAMPLE ERROR"); console.log(error); updatedMetadata[index].saved = false; - updatedMetadata[index].error = error; + updatedMetadata[index].error = error.response.data.error; }); } } } + return { metadata: updatedMetadata }; } ); From 9201e48e9d288a222d06065f90cd385dd2551005 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 21 Sep 2022 10:38:30 -0500 Subject: [PATCH 257/655] keeping formatted numeric cells instead of raw --- .../components/SampleMetadataImportUploadFile.jsx | 6 +++--- .../samples-metadata-import/services/importReducer.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx index 07df20bf84f..8dcd1a6706d 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx @@ -36,9 +36,9 @@ export function SampleMetadataImportUploadFile() { raw: true, }); const firstSheet = workbook.SheetNames[0]; - const rows = XLSX.utils.sheet_to_row_object_array( - workbook.Sheets[firstSheet] - ); + const rows = XLSX.utils.sheet_to_json(workbook.Sheets[firstSheet], { + rawNumbers: false, + }); dispatch(setHeaders(Object.keys(rows[0]))); dispatch(setMetadata(rows)); }; diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index a2fb3a0c8dc..b53aeb26d37 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -112,7 +112,7 @@ export const setSampleNameColumn = createAsyncThunk( isSampleNameValid: validateSampleName(metadataItem[column]), foundSampleId: response.data.samples .find((sample) => metadataItem[column] === sample.name) - .ids?.at(0), + .ids?.first(), saved: null, }; }); From 77702a94a01a8346da532184d8a6901288a43e6e Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 21 Sep 2022 12:46:00 -0500 Subject: [PATCH 258/655] fixing tests --- .../resources/js/apis/metadata/sample-utils.js | 8 ++++++-- .../services/importReducer.js | 16 +++++++++------- .../ProjectSampleMetadataImportPageIT.java | 11 +++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/main/webapp/resources/js/apis/metadata/sample-utils.js b/src/main/webapp/resources/js/apis/metadata/sample-utils.js index 0e4a7970e6d..b009bdbac0a 100644 --- a/src/main/webapp/resources/js/apis/metadata/sample-utils.js +++ b/src/main/webapp/resources/js/apis/metadata/sample-utils.js @@ -10,5 +10,9 @@ export const sampleNameRegex = new RegExp("^[A-Za-z0-9-_]{3,}$"); * @returns {boolean} */ export function validateSampleName(sampleName) { - return sampleNameRegex.test(sampleName); - } \ No newline at end of file + if (sampleName) { + return sampleNameRegex.test(sampleName); + } else { + return false; + } +} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index b53aeb26d37..b61d78f83d5 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -99,20 +99,22 @@ export const setSampleNameColumn = createAsyncThunk( const response = await validateSamples({ projectId: projectId, body: { - samples: metadata.map((row) => ({ - name: row[column], - })), + samples: metadata + .filter((row) => row[column]) + .map((row) => ({ + name: row[column], + })), }, }); - const updatedMetadata = metadata.map((metadataItem, index) => { + const foundSample = response.data.samples.find( + (sample) => metadataItem[column] === sample.name + ); return { ...metadataItem, rowKey: `metadata-uploader-row-${index}`, isSampleNameValid: validateSampleName(metadataItem[column]), - foundSampleId: response.data.samples - .find((sample) => metadataItem[column] === sample.name) - .ids?.first(), + foundSampleId: foundSample?.ids?.at(0), saved: null, }; }); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java index 63a5d944c87..cf1d4cd8fe2 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java @@ -48,9 +48,7 @@ public void testGoodFileAndHeaders() { */ List values = ImmutableList.of(2.222222, 2.3666, 1.5689, 63.89756, 59.6666) .stream() - .map(num -> BigDecimal.valueOf(num) - .setScale(2, RoundingMode.HALF_UP) - .doubleValue()) + .map(num -> BigDecimal.valueOf(num).setScale(2, RoundingMode.HALF_UP).doubleValue()) .collect(Collectors.toList()); List formattedNumbers = page.getValuesForColumnByName("Numbers"); formattedNumbers.forEach(num -> assertTrue(values.contains(Double.valueOf(num)), @@ -71,8 +69,8 @@ public void testMixedFileAndHeaders() { public void testSuccessfulUpload() { ProjectSampleMetadataImportPage page = ProjectSampleMetadataImportPage.goToPage(driver()); page.uploadMetadataFile(GOOD_FILE_PATH); - assertEquals("NLEP #", page.getValueForSelectedSampleNameColumn(), - "Has incorrect pre-populated sample name header"); + // assertEquals("NLEP #", page.getValueForSelectedSampleNameColumn(), + // "Has incorrect pre-populated sample name header"); page.goToReviewPage(); page.goToCompletePage(); assertTrue(page.isSuccessDisplayed(), "Success message did not display"); @@ -83,7 +81,8 @@ public void testSuccessfulUpload() { public void testFailedUpload() { ProjectSampleMetadataImportPage page = ProjectSampleMetadataImportPage.goToPage(driver()); page.uploadMetadataFile(INVALID_FILE_PATH); - page.goToReviewPage(); + page.selectSampleNameColumn(); + // page.goToReviewPage(); assertTrue(page.isAlertDisplayed(), "Validation message did not display"); } } From cfeafaa7a9b8891aa4d4bb78d3243ce321d45d8f Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 21 Sep 2022 14:00:42 -0500 Subject: [PATCH 259/655] fixing typescript errors --- .../resources/js/apis/projects/samples.ts | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index a970a5ebb32..529984d04c4 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -12,6 +12,13 @@ export interface SequencingFiles { pairs: PairedEndSequenceFile[]; } +export interface SampleRequest { + name: string; + organism: string; + description: string; + metadata: [{ field: string; value: string }]; +} + const PROJECT_ID = getProjectIdFromUrl(); const URL = setBaseUrl(`/ajax/projects`); @@ -71,15 +78,35 @@ export const { useShareSamplesWithProjectMutation, } = samplesApi; -export async function validateSamples({ projectId, body }) { +export async function validateSamples({ + projectId, + body, +}: { + projectId: number; + body: SampleRequest; +}) { return await axios.post(`${URL}/${projectId}/samples/validate`, body); } -export async function createSample({ projectId, body }) { +export async function createSample({ + projectId, + body, +}: { + projectId: number; + body: SampleRequest; +}) { return await axios.post(`${URL}/${projectId}/samples/add-sample`, body); } -export async function updateSample({ projectId, sampleId, body }) { +export async function updateSample({ + projectId, + sampleId, + body, +}: { + projectId: number; + sampleId: number; + body: SampleRequest; +}) { return await axios.patch( `${URL}/${projectId}/samples/add-sample/${sampleId}`, body From 2fcfe852181cd67fa37632d5b4b089fb6ed0bd2b Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 21 Sep 2022 14:34:05 -0500 Subject: [PATCH 260/655] adding table pagination to review table --- .../components/SampleMetadataImportReview.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 45afc920b71..8437c9da324 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -11,6 +11,7 @@ import { red1, red2, red5 } from "../../../../styles/colors"; import styled from "styled-components"; import { useDispatch, useSelector } from "react-redux"; import { saveMetadata } from "../services/importReducer"; +import { getPaginationOptions } from "../../../../utilities/antdesign-table-utilities"; const { Paragraph, Text } = Typography; @@ -192,7 +193,7 @@ export function SampleMetadataImportReview() { columns={columns} dataSource={metadata} scroll={{ x: "max-content", y: 600 }} - pagination={false} + pagination={getPaginationOptions(metadata.length)} />
    From 11c6ffa11497742e1c58951d8042d9463690b95c Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 22 Sep 2022 15:07:32 -0500 Subject: [PATCH 261/655] adding progress bar for saving samples --- .../components/SampleMetadataImportReview.jsx | 11 ++++++--- .../components/SampleMetadataImportSteps.jsx | 13 ++++++++-- .../components/SampleMetadataImportWizard.jsx | 3 +++ .../services/importReducer.js | 24 ++++++++++++++++++- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 8437c9da324..53c05792042 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -48,9 +48,11 @@ export function SampleMetadataImportReview() { const [columns, setColumns] = React.useState([]); const [selected, setSelected] = React.useState([]); const [valid, setValid] = React.useState(true); - const { headers, sampleNameColumn, metadata } = useSelector( + const [progress, setProgress] = React.useState(0); + const { headers, sampleNameColumn, metadata, savedCount } = useSelector( (state) => state.importReducer ); + const dispatch = useDispatch(); const tagColumn = { @@ -95,6 +97,10 @@ export function SampleMetadataImportReview() { }), }; + React.useEffect(() => { + setProgress((savedCount / selected.length) * 100); + }, [savedCount]); + React.useEffect(() => { setValid(!metadata.some((row) => row.isSampleNameValid === false)); @@ -166,7 +172,7 @@ export function SampleMetadataImportReview() { }; return ( - + {i18n("SampleMetadataImportReview.description")} {!valid && ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx index c5c5bd16781..92a2bb688b4 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx @@ -7,12 +7,21 @@ const { Step } = Steps; * React component that displays the steps for the Sample Metadata Uploader. * @prop {number} currentStep - the current step, starting with zero * @prop {string} currentStatus - the status of the current step + * @prop {string} currentPercent - the progress percentage of the current step * @returns {*} * @constructor */ -export function SampleMetadataImportSteps({ currentStep, currentStatus }) { +export function SampleMetadataImportSteps({ + currentStep, + currentStatus, + currentPercent, +}) { return ( - + diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx index 3e77513b8a5..0038c396d7f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx @@ -8,6 +8,7 @@ import { setBaseUrl } from "../../../../utilities/url-utilities"; * React component that displays the Sample Metadata Uploader Wizard wrapper. * @prop {number} currentStep - the current step, starting with zero * @prop {string} currentStatus - the status of the current step + * @prop {string} currentPercent - the progress percentage of the current step * @prop {any} children - the status of the current step * @returns {*} * @constructor @@ -15,6 +16,7 @@ import { setBaseUrl } from "../../../../utilities/url-utilities"; export function SampleMetadataImportWizard({ currentStep, currentStatus, + currentPercent, children, }) { const { projectId } = useParams(); @@ -31,6 +33,7 @@ export function SampleMetadataImportWizard({ {children} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index b61d78f83d5..8999fe51fd6 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -14,6 +14,7 @@ const initialState = { sampleNameColumn: "", headers: [], metadata: [], + savedCount: 0, }; /* @@ -22,7 +23,7 @@ For more information on redux async thunks see: https://redux-toolkit.js.org/api */ export const saveMetadata = createAsyncThunk( `importReducer/saveMetadata`, - async ({ projectId, selectedMetadataKeys }, { getState }) => { + async ({ projectId, selectedMetadataKeys }, { dispatch, getState }) => { const state = getState(); const sampleNameColumn = state.importReducer.sampleNameColumn; const headers = state.importReducer.headers; @@ -80,6 +81,13 @@ export const saveMetadata = createAsyncThunk( updatedMetadata[index].error = error.response.data.error; }); } + dispatch( + setSavedCount( + updatedMetadata.filter( + (updatedMetadataItem) => updatedMetadataItem.saved + ).length + ) + ); } } @@ -145,6 +153,17 @@ export const setMetadata = createAction( }) ); +/* +Redux action for setting the project metadata. +For more information on redux actions see: https://redux-toolkit.js.org/api/createAction + */ +export const setSavedCount = createAction( + `importReducer/setSavedCount`, + (savedCount) => ({ + payload: { savedCount }, + }) +); + /* Redux reducer for project metadata. For more information on redux reducers see: https://redux-toolkit.js.org/api/createReducer @@ -156,6 +175,9 @@ export const importReducer = createReducer(initialState, (builder) => { builder.addCase(setMetadata, (state, action) => { state.metadata = action.payload.metadata; }); + builder.addCase(setSavedCount, (state, action) => { + state.savedCount = action.payload.savedCount; + }); builder.addCase(setSampleNameColumn.fulfilled, (state, action) => { state.sampleNameColumn = action.payload.sampleNameColumn; state.metadata = action.payload.metadata; From 22d4aea0509de30b1eacdc814bd41e71575ad403 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 23 Sep 2022 12:10:15 -0500 Subject: [PATCH 262/655] trying chunk uploading --- .../services/importReducer.js | 113 ++++++++++-------- 1 file changed, 62 insertions(+), 51 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 8999fe51fd6..590840c3166 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -30,57 +30,68 @@ export const saveMetadata = createAsyncThunk( const metadata = state.importReducer.metadata; const updatedMetadata = JSON.parse(JSON.stringify(metadata)); - for (const [index, metadataItem] of updatedMetadata.entries()) { - if (selectedMetadataKeys.includes(metadataItem.rowKey)) { - const name = metadataItem[sampleNameColumn]; - const metadataFields = Object.entries(metadataItem) - .filter( - ([key, value]) => headers.includes(key) && key !== sampleNameColumn - ) - .map(([key, value]) => ({ field: key, value })); - const sampleId = metadataItem.foundSampleId; - - if (sampleId) { - await updateSample({ - projectId, - sampleId, - body: { - name, - // TODO: Don't overwrite organism & description - metadata: metadataFields, - }, - }) - .then((response) => { - console.log("UPDATE SAMPLE RESPONSE"); - console.log(response); - updatedMetadata[index].saved = true; - }) - .catch((error) => { - console.log("UPDATE SAMPLE ERROR"); - console.log(error); - updatedMetadata[index].saved = false; - updatedMetadata[index].error = error.response.data.error; - }); - } else { - await createSample({ - projectId, - body: { - name, - metadata: metadataFields, - }, - }) - .then((response) => { - console.log("CREATE SAMPLE RESPONSE"); - console.log(response); - updatedMetadata[index].saved = true; - }) - .catch((error) => { - console.log("CREATE SAMPLE ERROR"); - console.log(error); - updatedMetadata[index].saved = false; - updatedMetadata[index].error = error.response.data.error; - }); + const chunkSize = 100; + for (let i = 0; i < metadata.length; i = i + chunkSize) { + const promises = []; + for (let j = i; j < i + chunkSize && j < metadata.length; j++) { + const metadataItem = metadata[j]; + if (selectedMetadataKeys.includes(metadataItem.rowKey)) { + const name = metadataItem[sampleNameColumn]; + const metadataFields = Object.entries(metadataItem) + .filter( + ([key, value]) => + headers.includes(key) && key !== sampleNameColumn + ) + .map(([key, value]) => ({ field: key, value })); + const sampleId = metadataItem.foundSampleId; + if (sampleId) { + promises.push( + updateSample({ + projectId, + sampleId, + body: { + name, + // TODO: Don't overwrite organism & description + metadata: metadataFields, + }, + }) + .then((response) => { + console.log("UPDATE SAMPLE RESPONSE"); + console.log(response); + updatedMetadata[j].saved = true; + }) + .catch((error) => { + console.log("UPDATE SAMPLE ERROR"); + console.log(error); + updatedMetadata[j].saved = false; + updatedMetadata[j].error = error.response.data.error; + }) + ); + } else { + promises.push( + createSample({ + projectId, + body: { + name, + metadata: metadataFields, + }, + }) + .then((response) => { + console.log("CREATE SAMPLE RESPONSE"); + console.log(response); + updatedMetadata[j].saved = true; + }) + .catch((error) => { + console.log("CREATE SAMPLE ERROR"); + console.log(error); + updatedMetadata[j].saved = false; + updatedMetadata[j].error = error.response.data.error; + }) + ); + } } + } + await Promise.all(promises).then(() => { dispatch( setSavedCount( updatedMetadata.filter( @@ -88,7 +99,7 @@ export const saveMetadata = createAsyncThunk( ).length ) ); - } + }); } return { metadata: updatedMetadata }; From a87f6156ca8032a93561183bf73cc3e6701dd9bb Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 23 Sep 2022 13:09:09 -0500 Subject: [PATCH 263/655] fixing test --- .../projects/ProjectSampleMetadataImportPageIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java index cf1d4cd8fe2..9f3e2e0e50d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java @@ -71,10 +71,10 @@ public void testSuccessfulUpload() { page.uploadMetadataFile(GOOD_FILE_PATH); // assertEquals("NLEP #", page.getValueForSelectedSampleNameColumn(), // "Has incorrect pre-populated sample name header"); - page.goToReviewPage(); + // page.goToReviewPage(); + page.selectSampleNameColumn(); page.goToCompletePage(); assertTrue(page.isSuccessDisplayed(), "Success message did not display"); - } @Test From e20bc82b222a3c61069799e7265db09fde633dfa Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 27 Sep 2022 10:17:42 -0500 Subject: [PATCH 264/655] adding some spinners to pages to show loading --- .../SampleMetadataImportMapHeaders.jsx | 3 +++ .../components/SampleMetadataImportReview.jsx | 5 ++++ .../SampleMetadataImportUploadFile.jsx | 23 +++++++++++-------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx index 8b0fcf4cd46..78521fe8b7e 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx @@ -22,6 +22,7 @@ export function SampleMetadataImportMapHeaders() { const { projectId } = useParams(); const navigate = useNavigate(); const [column, setColumn] = React.useState(); + const [loading, setLoading] = React.useState(false); const { headers, sampleNameColumn } = useSelector( (state) => state.importReducer ); @@ -34,6 +35,7 @@ export function SampleMetadataImportMapHeaders() { }, [sampleNameColumn, headers]); const onSubmit = async () => { + setLoading(true); await dispatch(setSampleNameColumn({ projectId, column })); navigate(`/${projectId}/sample-metadata/upload/review`); }; @@ -69,6 +71,7 @@ export function SampleMetadataImportMapHeaders() { className="t-metadata-uploader-preview-button" onClick={onSubmit} style={{ marginLeft: "auto" }} + loading={loading} > {i18n("SampleMetadataImportMapHeaders.button.next")} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index 53c05792042..ee3ac61163e 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -49,6 +49,7 @@ export function SampleMetadataImportReview() { const [selected, setSelected] = React.useState([]); const [valid, setValid] = React.useState(true); const [progress, setProgress] = React.useState(0); + const [loading, setLoading] = React.useState(false); const { headers, sampleNameColumn, metadata, savedCount } = useSelector( (state) => state.importReducer ); @@ -156,6 +157,7 @@ export function SampleMetadataImportReview() { }, []); const save = async () => { + setLoading(true); const selectedMetadataKeys = metadata .filter((metadataItem) => selected.includes(metadataItem.rowKey)) .map((metadataItem) => metadataItem.rowKey); @@ -168,6 +170,8 @@ export function SampleMetadataImportReview() { response.payload.metadata.every((metadataItem) => !metadataItem.error) ) { navigate(`/${projectId}/sample-metadata/upload/complete`); + } else { + setLoading(false); } }; @@ -213,6 +217,7 @@ export function SampleMetadataImportReview() { className="t-metadata-uploader-upload-button" style={{ marginLeft: "auto" }} onClick={save} + loading={loading} > {i18n("SampleMetadataImportReview.button.next")} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx index 8dcd1a6706d..c060bdff744 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.jsx @@ -2,7 +2,7 @@ import React from "react"; import { useDispatch } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; import { setHeaders, setMetadata } from "../services/importReducer"; -import { notification, Typography } from "antd"; +import { notification, Spin, Typography } from "antd"; import { DragUpload } from "../../../../components/files/DragUpload"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import * as XLSX from "xlsx"; @@ -20,6 +20,7 @@ export function SampleMetadataImportUploadFile() { const navigate = useNavigate(); const dispatch = useDispatch(); const [status, setStatus] = React.useState("process"); + const [loading, setLoading] = React.useState(false); const options = { multiple: false, @@ -28,6 +29,7 @@ export function SampleMetadataImportUploadFile() { onChange(info) { const { status } = info.file; if (status !== "uploading") { + setLoading(true); let reader = new FileReader(); if (reader.readAsBinaryString) { reader.onload = (e) => { @@ -54,6 +56,7 @@ export function SampleMetadataImportUploadFile() { }); navigate(`/${projectId}/sample-metadata/upload/headers`); } else if (status === "error") { + setLoading(false); setStatus("error"); notification.error({ message: i18n("SampleMetadataImportUploadFile.error", info.file.name), @@ -64,14 +67,16 @@ export function SampleMetadataImportUploadFile() { return ( - {i18n("SampleMetadataImportUploadFile.warning")} - } - options={options} - /> + + {i18n("SampleMetadataImportUploadFile.warning")} + } + options={options} + /> + ); } From 9dd9e6080ba8914db05ebd2db02b369da152625e Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 27 Sep 2022 10:36:39 -0500 Subject: [PATCH 265/655] navigating back to beginning after upload complete --- .../components/SampleMetadataImportComplete.jsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx index 6ae6ed2de2c..9cd5160e8f6 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx @@ -1,5 +1,5 @@ import React from "react"; -import { useParams } from "react-router-dom"; +import { useNavigate, useParams } from "react-router-dom"; import { Button, Result } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { setBaseUrl } from "../../../../utilities/url-utilities"; @@ -13,6 +13,7 @@ import { useSelector } from "react-redux"; */ export function SampleMetadataImportComplete() { const { metadata } = useSelector((state) => state.importReducer); + const samplesUpdatedCount = metadata.filter( (metadataItem) => metadataItem.saved === true && metadataItem.foundSampleId ).length; @@ -43,6 +44,13 @@ export function SampleMetadataImportComplete() { ); const { projectId } = useParams(); + const navigate = useNavigate(); + + React.useEffect(() => { + setTimeout(() => { + navigate(`/${projectId}/sample-metadata/upload/file`); + }, 5000); + }, []); return ( From 12a235420664c98375d64b982bdc3ea1005ae3ab Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 27 Sep 2022 17:02:47 -0500 Subject: [PATCH 266/655] Removed dependency no longer required --- .../bioinformatics/irida/config/web/IridaUIWebConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java index 3c78831499a..f384bc18281 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java @@ -272,7 +272,7 @@ public void addArgumentResolvers(List argumentRes */ private Set additionalDialects() { Set dialects = new HashSet<>(); - dialects.add(new WebpackerDialect(!env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); + dialects.add(new WebpackerDialect(env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); dialects.add(new SpringSecurityDialect()); dialects.add(new LayoutDialect()); dialects.add(new DataAttributeDialect()); From 3094c7271a67d1b79ea297c9859b1487eb43ae61 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 27 Sep 2022 17:04:09 -0500 Subject: [PATCH 267/655] Reverting change to pushed up file and pushing up gradle file with removed dependency --- build.gradle.kts | 3 --- .../bioinformatics/irida/config/web/IridaUIWebConfig.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 74c8215a23a..74bf909f7ac 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -175,9 +175,6 @@ dependencies { exclude(group = "jakarta.xml.bind", module = "jakarta.xml.bind-api") exclude(group = "jakarta.validation", module = "jakarta.validation-api") } - implementation("com.monitorjbl:xlsx-streamer:2.1.0") { - exclude(group = "xml-apis", module = "xml-apis") - } // Microsoft Azure implementation("com.azure:azure-storage-blob:12.18.0") { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java index f384bc18281..3c78831499a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java @@ -272,7 +272,7 @@ public void addArgumentResolvers(List argumentRes */ private Set additionalDialects() { Set dialects = new HashSet<>(); - dialects.add(new WebpackerDialect(env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); + dialects.add(new WebpackerDialect(!env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); dialects.add(new SpringSecurityDialect()); dialects.add(new LayoutDialect()); dialects.add(new DataAttributeDialect()); From facf54f4d242b9c62c2babd936a9ea9f8f1474c1 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 28 Sep 2022 12:34:58 -0500 Subject: [PATCH 268/655] storing saving and validation details in seperate hashes instead of modifying the metadata object where twp objects are stored in memory temporarily --- .../SampleMetadataImportComplete.jsx | 13 ++- .../components/SampleMetadataImportReview.jsx | 61 +++++++++---- .../services/importReducer.js | 86 ++++++++++--------- 3 files changed, 99 insertions(+), 61 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx index 9cd5160e8f6..a63153251fd 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx @@ -12,14 +12,19 @@ import { useSelector } from "react-redux"; * @constructor */ export function SampleMetadataImportComplete() { - const { metadata } = useSelector((state) => state.importReducer); + const { metadata, metadataValidateDetails, metadataSaveDetails } = + useSelector((state) => state.importReducer); const samplesUpdatedCount = metadata.filter( - (metadataItem) => metadataItem.saved === true && metadataItem.foundSampleId + (metadataItem) => + metadataSaveDetails[metadataItem.rowKey]?.saved === true && + metadataValidateDetails[metadataItem.rowKey].foundSampleId ).length; const samplesCreatedCount = metadata.filter( - (metadataItem) => metadataItem.saved === true && !metadataItem.foundSampleId + (metadataItem) => + metadataSaveDetails[metadataItem.rowKey]?.saved === true && + !metadataValidateDetails[metadataItem.rowKey].foundSampleId ).length; let stats = @@ -49,7 +54,7 @@ export function SampleMetadataImportComplete() { React.useEffect(() => { setTimeout(() => { navigate(`/${projectId}/sample-metadata/upload/file`); - }, 5000); + }, 10000); }, []); return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx index ee3ac61163e..1ea5b932b32 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx @@ -50,10 +50,13 @@ export function SampleMetadataImportReview() { const [valid, setValid] = React.useState(true); const [progress, setProgress] = React.useState(0); const [loading, setLoading] = React.useState(false); - const { headers, sampleNameColumn, metadata, savedCount } = useSelector( - (state) => state.importReducer - ); - + const { + headers, + sampleNameColumn, + metadata, + metadataValidateDetails, + metadataSaveDetails, + } = useSelector((state) => state.importReducer); const dispatch = useDispatch(); const tagColumn = { @@ -63,7 +66,7 @@ export function SampleMetadataImportReview() { fixed: "left", width: 70, render: (text, item) => { - if (!item.foundSampleId) + if (!metadataValidateDetails[item.rowKey].foundSampleId) return ( {i18n("SampleMetadataImportReview.table.filter.new")} @@ -81,7 +84,9 @@ export function SampleMetadataImportReview() { }, ], onFilter: (value, record) => - value === "new" ? !record.foundSampleId : record.foundSampleId, + value === "new" + ? !metadataValidateDetails[record.rowKey].foundSampleId + : metadataValidateDetails[record.rowKey].foundSampleId, }; const rowSelection = { @@ -92,18 +97,25 @@ export function SampleMetadataImportReview() { }, getCheckboxProps: (record) => ({ disabled: !( - record.isSampleNameValid && - (record.saved === null || record.saved === true) + metadataValidateDetails[record.rowKey].isSampleNameValid || + metadataSaveDetails[record.rowKey]?.saved === true ), }), }; React.useEffect(() => { + const savedCount = Object.entries(metadataSaveDetails).filter( + ([key, metadataSaveDetailsItem]) => metadataSaveDetailsItem.saved + ).length; setProgress((savedCount / selected.length) * 100); - }, [savedCount]); + }, [metadataSaveDetails]); React.useEffect(() => { - setValid(!metadata.some((row) => row.isSampleNameValid === false)); + setValid( + !metadata.some( + (row) => metadataValidateDetails[row.rowKey].isSampleNameValid === false + ) + ); const sampleColumn = { title: sampleNameColumn, @@ -113,7 +125,10 @@ export function SampleMetadataImportReview() { onCell: (item) => { return { style: { - background: item.isSampleNameValid === true ? null : red1, + background: + metadataValidateDetails[item.rowKey].isSampleNameValid === true + ? null + : red1, }, }; }, @@ -124,9 +139,12 @@ export function SampleMetadataImportReview() { fixed: "left", width: 10, render: (text, item) => { - if (item.saved === false) + if (metadataSaveDetails[item.rowKey]?.saved === false) return ( - + ); @@ -150,11 +168,14 @@ export function SampleMetadataImportReview() { setColumns(updatedColumns); setSelected( metadata.map((row) => { - if (row.isSampleNameValid && (row.saved === null || row.saved === true)) + if ( + metadataValidateDetails[row.rowKey].isSampleNameValid || + metadataSaveDetails[row.rowKey]?.saved === true + ) return row.rowKey; }) ); - }, []); + }, [metadataSaveDetails]); const save = async () => { setLoading(true); @@ -167,7 +188,9 @@ export function SampleMetadataImportReview() { ); if ( - response.payload.metadata.every((metadataItem) => !metadataItem.error) + Object.entries(response.payload.metadataSaveDetails).filter( + ([, metadataSaveDetailsItem]) => metadataSaveDetailsItem.error + ).length === 0 ) { navigate(`/${projectId}/sample-metadata/upload/complete`); } else { @@ -198,7 +221,11 @@ export function SampleMetadataImportReview() { row.rowKey} - rowClassName={(record) => (record.saved === false ? "row-error" : null)} + rowClassName={(record) => + metadataSaveDetails[record.rowKey]?.saved === false + ? "row-error" + : null + } rowSelection={rowSelection} columns={columns} dataSource={metadata} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 590840c3166..627c8c119d2 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -14,7 +14,8 @@ const initialState = { sampleNameColumn: "", headers: [], metadata: [], - savedCount: 0, + metadataValidateDetails: {}, + metadataSaveDetails: {}, }; /* @@ -28,13 +29,15 @@ export const saveMetadata = createAsyncThunk( const sampleNameColumn = state.importReducer.sampleNameColumn; const headers = state.importReducer.headers; const metadata = state.importReducer.metadata; - const updatedMetadata = JSON.parse(JSON.stringify(metadata)); + const metadataValidateDetails = state.importReducer.metadataValidateDetails; + const metadataSaveDetails = {}; const chunkSize = 100; for (let i = 0; i < metadata.length; i = i + chunkSize) { const promises = []; for (let j = i; j < i + chunkSize && j < metadata.length; j++) { const metadataItem = metadata[j]; + const index = metadataItem.rowKey; if (selectedMetadataKeys.includes(metadataItem.rowKey)) { const name = metadataItem[sampleNameColumn]; const metadataFields = Object.entries(metadataItem) @@ -43,7 +46,8 @@ export const saveMetadata = createAsyncThunk( headers.includes(key) && key !== sampleNameColumn ) .map(([key, value]) => ({ field: key, value })); - const sampleId = metadataItem.foundSampleId; + const sampleId = + metadataValidateDetails[metadataItem.rowKey].foundSampleId; if (sampleId) { promises.push( updateSample({ @@ -56,15 +60,13 @@ export const saveMetadata = createAsyncThunk( }, }) .then((response) => { - console.log("UPDATE SAMPLE RESPONSE"); - console.log(response); - updatedMetadata[j].saved = true; + metadataSaveDetails[index] = { saved: true }; }) .catch((error) => { - console.log("UPDATE SAMPLE ERROR"); - console.log(error); - updatedMetadata[j].saved = false; - updatedMetadata[j].error = error.response.data.error; + metadataSaveDetails[index] = { + saved: false, + error: error.response.data.error, + }; }) ); } else { @@ -77,15 +79,13 @@ export const saveMetadata = createAsyncThunk( }, }) .then((response) => { - console.log("CREATE SAMPLE RESPONSE"); - console.log(response); - updatedMetadata[j].saved = true; + metadataSaveDetails[index] = { saved: true }; }) .catch((error) => { - console.log("CREATE SAMPLE ERROR"); - console.log(error); - updatedMetadata[j].saved = false; - updatedMetadata[j].error = error.response.data.error; + metadataSaveDetails[index] = { + saved: false, + error: error.response.data.error, + }; }) ); } @@ -93,16 +93,12 @@ export const saveMetadata = createAsyncThunk( } await Promise.all(promises).then(() => { dispatch( - setSavedCount( - updatedMetadata.filter( - (updatedMetadataItem) => updatedMetadataItem.saved - ).length - ) + setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) ); }); } - return { metadata: updatedMetadata }; + return { metadataSaveDetails }; } ); @@ -115,6 +111,7 @@ export const setSampleNameColumn = createAsyncThunk( async ({ projectId, column }, { getState }) => { const state = getState(); const metadata = state.importReducer.metadata; + const metadataValidateDetails = {}; const response = await validateSamples({ projectId: projectId, body: { @@ -125,20 +122,22 @@ export const setSampleNameColumn = createAsyncThunk( })), }, }); - const updatedMetadata = metadata.map((metadataItem, index) => { + for (let i = 0; i < metadata.length; i++) { + const metadataItem = metadata[i]; + const index = metadataItem.rowKey; const foundSample = response.data.samples.find( (sample) => metadataItem[column] === sample.name ); - return { - ...metadataItem, - rowKey: `metadata-uploader-row-${index}`, + metadataValidateDetails[index] = { isSampleNameValid: validateSampleName(metadataItem[column]), foundSampleId: foundSample?.ids?.at(0), - saved: null, }; - }); + } - return { sampleNameColumn: column, metadata: updatedMetadata }; + return { + sampleNameColumn: column, + metadataValidateDetails, + }; } ); @@ -160,18 +159,25 @@ For more information on redux actions see: https://redux-toolkit.js.org/api/crea export const setMetadata = createAction( `importReducer/setMetadata`, (metadata) => ({ - payload: { metadata }, + payload: { + metadata: metadata.map((metadataItem, index) => { + return { + ...metadataItem, + rowKey: `metadata-uploader-row-${index}`, + }; + }), + }, }) ); /* -Redux action for setting the project metadata. +Redux action for setting the project metadata save details. For more information on redux actions see: https://redux-toolkit.js.org/api/createAction */ -export const setSavedCount = createAction( - `importReducer/setSavedCount`, - (savedCount) => ({ - payload: { savedCount }, +export const setMetadataSaveDetails = createAction( + `importReducer/setMetadataSaveDetails`, + (metadataSaveDetails) => ({ + payload: { metadataSaveDetails }, }) ); @@ -186,14 +192,14 @@ export const importReducer = createReducer(initialState, (builder) => { builder.addCase(setMetadata, (state, action) => { state.metadata = action.payload.metadata; }); - builder.addCase(setSavedCount, (state, action) => { - state.savedCount = action.payload.savedCount; + builder.addCase(setMetadataSaveDetails, (state, action) => { + state.metadataSaveDetails = action.payload.metadataSaveDetails; }); builder.addCase(setSampleNameColumn.fulfilled, (state, action) => { state.sampleNameColumn = action.payload.sampleNameColumn; - state.metadata = action.payload.metadata; + state.metadataValidateDetails = action.payload.metadataValidateDetails; }); builder.addCase(saveMetadata.fulfilled, (state, action) => { - state.metadata = action.payload.metadata; + state.metadataSaveDetails = action.payload.metadataSaveDetails; }); }); From 4b8c1d6b66090a8cc298ce3f868c789e223b9271 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 28 Sep 2022 15:02:17 -0500 Subject: [PATCH 269/655] skipping saving samples that have already been saved --- .../samples-metadata-import/services/importReducer.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js index 627c8c119d2..70f6d405344 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js @@ -38,7 +38,10 @@ export const saveMetadata = createAsyncThunk( for (let j = i; j < i + chunkSize && j < metadata.length; j++) { const metadataItem = metadata[j]; const index = metadataItem.rowKey; - if (selectedMetadataKeys.includes(metadataItem.rowKey)) { + if ( + selectedMetadataKeys.includes(index) && + metadataSaveDetails[index]?.saved !== true + ) { const name = metadataItem[sampleNameColumn]; const metadataFields = Object.entries(metadataItem) .filter( @@ -46,8 +49,7 @@ export const saveMetadata = createAsyncThunk( headers.includes(key) && key !== sampleNameColumn ) .map(([key, value]) => ({ field: key, value })); - const sampleId = - metadataValidateDetails[metadataItem.rowKey].foundSampleId; + const sampleId = metadataValidateDetails[index].foundSampleId; if (sampleId) { promises.push( updateSample({ From d47b7ff4115cd32cd7225c7d042e429aef178f64 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 28 Sep 2022 15:50:30 -0500 Subject: [PATCH 270/655] converting SampleMetadataImportSteps.jsx to typescript --- ...teps.jsx => SampleMetadataImportSteps.tsx} | 21 ++++++++----------- .../components/SampleMetadataImportWizard.jsx | 6 +++--- 2 files changed, 12 insertions(+), 15 deletions(-) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportSteps.jsx => SampleMetadataImportSteps.tsx} (58%) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.tsx similarity index 58% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.tsx index 92a2bb688b4..497bbd0af70 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSteps.tsx @@ -1,27 +1,24 @@ import React from "react"; import { Steps } from "antd"; +import { StepsProps } from "antd/lib/steps"; const { Step } = Steps; /** * React component that displays the steps for the Sample Metadata Uploader. - * @prop {number} currentStep - the current step, starting with zero - * @prop {string} currentStatus - the status of the current step - * @prop {string} currentPercent - the progress percentage of the current step + * @prop current - the current step, starting with zero + * @prop status - the status of the current step + * @prop percent - the progress percentage of the current step * @returns {*} * @constructor */ export function SampleMetadataImportSteps({ - currentStep, - currentStatus, - currentPercent, -}) { + current, + status, + percent, +}: StepsProps): JSX.Element { return ( - + diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx index 0038c396d7f..bf6c1670204 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.jsx @@ -31,9 +31,9 @@ export function SampleMetadataImportWizard({ } /> {children} From 29612d2f752dc5dc37053a976396b16a303c13f4 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 29 Sep 2022 13:01:05 -0500 Subject: [PATCH 271/655] converting SampleMetadataImportUploadFile.jsx to typescript --- .../files/{DragUpload.jsx => DragUpload.tsx} | 19 +++++++++-- .../launch/references/UploadReferenceFile.jsx | 4 +-- .../SampleMetadataImportComplete.jsx | 2 +- .../SampleMetadataImportMapHeaders.jsx | 2 +- .../components/SampleMetadataImportReview.jsx | 2 +- .../components/SampleMetadataImportSteps.tsx | 3 +- ...jsx => SampleMetadataImportUploadFile.tsx} | 32 +++++++++++-------- ...ard.jsx => SampleMetadataImportWizard.tsx} | 31 +++++++++++------- 8 files changed, 61 insertions(+), 34 deletions(-) rename src/main/webapp/resources/js/components/files/{DragUpload.jsx => DragUpload.tsx} (58%) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportUploadFile.jsx => SampleMetadataImportUploadFile.tsx} (71%) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportWizard.jsx => SampleMetadataImportWizard.tsx} (58%) diff --git a/src/main/webapp/resources/js/components/files/DragUpload.jsx b/src/main/webapp/resources/js/components/files/DragUpload.tsx similarity index 58% rename from src/main/webapp/resources/js/components/files/DragUpload.jsx rename to src/main/webapp/resources/js/components/files/DragUpload.tsx index 29633c2ec4a..7e388a1b21e 100644 --- a/src/main/webapp/resources/js/components/files/DragUpload.jsx +++ b/src/main/webapp/resources/js/components/files/DragUpload.tsx @@ -1,18 +1,33 @@ import React from "react"; +import type { UploadProps } from "antd"; import { Upload } from "antd"; import { IconFileUpload } from "../icons/Icons"; import { SPACE_SM } from "../../styles/spacing"; const { Dragger } = Upload; +interface DragUploadProps extends React.HTMLAttributes { + uploadText?: string; + uploadHint?: React.ReactElement; + options?: UploadProps; +} + /** * React component for rendering the drag and drop upload functionality. - * @param {object} - upload options as well as text/hint for drag and drop + * @param uploadText - text for drag and drop + * @param uploadHint - hint for drag and drop + * @param options - upload options + * @param props - remainder of props passed * @returns {*} * @constructor */ -export function DragUpload({ uploadText, uploadHint, options, ...props }) { +export function DragUpload({ + uploadText, + uploadHint, + options, + ...props +}: DragUploadProps): JSX.Element { return (
    diff --git a/src/main/webapp/resources/js/pages/launch/references/UploadReferenceFile.jsx b/src/main/webapp/resources/js/pages/launch/references/UploadReferenceFile.jsx index 85519289499..eac13a5e4f6 100644 --- a/src/main/webapp/resources/js/pages/launch/references/UploadReferenceFile.jsx +++ b/src/main/webapp/resources/js/pages/launch/references/UploadReferenceFile.jsx @@ -1,7 +1,7 @@ import React from "react"; import { notification } from "antd"; import { setBaseUrl } from "../../../utilities/url-utilities"; -import { DragUpload } from "../../../components/files/DragUpload.jsx"; +import { DragUpload } from "../../../components/files/DragUpload.tsx"; import { useLaunch } from "../launch-context"; import { referenceFileUploadComplete } from "../launch-dispatch"; @@ -32,7 +32,7 @@ export function UploadReferenceFile({ form }) { } else if (status === "error") { notification.error({ message: i18n("ReferenceFile.uploadFileError", info.file.name), - description: response.error + description: response.error, }); } }, diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx index a63153251fd..4d283f9e37b 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx @@ -58,7 +58,7 @@ export function SampleMetadataImportComplete() { }, []); return ( - + + {i18n("SampleMetadataImportMapHeaders.description")} + {i18n("SampleMetadataImportReview.description")} {!valid && ( ("process"); + const [loading, setLoading] = React.useState(false); - const options = { + const options: UploadProps = { multiple: false, showUploadList: false, - accept: [".xls", ".xlsx", ".csv"], - onChange(info) { + accept: ".xls,.xlsx,.csv", + onChange(info: { + file: { originFileObj?: any; name?: any; status?: any }; + }) { const { status } = info.file; if (status !== "uploading") { setLoading(true); let reader = new FileReader(); if (reader.readAsBinaryString) { reader.onload = (e) => { - const workbook = XLSX.read(reader.result, { + const workbook: WorkBook = XLSX.read(reader.result, { type: "binary", raw: true, }); - const firstSheet = workbook.SheetNames[0]; - const rows = XLSX.utils.sheet_to_json(workbook.Sheets[firstSheet], { - rawNumbers: false, - }); + const firstSheet: string = workbook.SheetNames[0]; + const rows: any[] = XLSX.utils.sheet_to_json( + workbook.Sheets[firstSheet], + { + rawNumbers: false, + } + ); dispatch(setHeaders(Object.keys(rows[0]))); dispatch(setMetadata(rows)); }; @@ -66,7 +72,7 @@ export function SampleMetadataImportUploadFile() { }; return ( - + {children} From ef985c4590c58c2d678d29d52262afd7702effa7 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 4 Oct 2022 10:18:49 -0500 Subject: [PATCH 272/655] converting SampleMetadataImportMapHeaders.jsx to typescript --- .../resources/js/apis/projects/samples.ts | 42 +++++-- ...lockRadioInput.jsx => BlockRadioInput.tsx} | 10 +- ...jsx => SampleMetadataImportMapHeaders.tsx} | 30 ++--- .../SampleMetadataImportUploadFile.tsx | 8 +- .../components/SampleMetadataImportWizard.tsx | 2 +- .../{importReducer.js => importReducer.ts} | 108 ++++++++++++------ .../{store.js => store.ts} | 7 +- 7 files changed, 149 insertions(+), 58 deletions(-) rename src/main/webapp/resources/js/components/ant.design/forms/{BlockRadioInput.jsx => BlockRadioInput.tsx} (76%) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportMapHeaders.jsx => SampleMetadataImportMapHeaders.tsx} (69%) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/{importReducer.js => importReducer.ts} (64%) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/{store.js => store.ts} (68%) diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index 529984d04c4..3faa6292af8 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -12,11 +12,35 @@ export interface SequencingFiles { pairs: PairedEndSequenceFile[]; } +export interface ValidateSampleNameModel { + ids?: number[]; + name: string; +} + +export interface ValidateSamplesResponse { + samples: ValidateSampleNameModel[]; +} + +export interface MetadataItem { + [field: string]: string; + rowKey: string; +} + +export interface FieldUpdate { + field: string; + value: string; +} + export interface SampleRequest { name: string; - organism: string; - description: string; - metadata: [{ field: string; value: string }]; + organism?: string; + description?: string; + metadata: FieldUpdate[]; +} + +export interface ValidateSampleNamesRequest { + samples: ValidateSampleNameModel[]; + associatedProjectIds?: number[]; } const PROJECT_ID = getProjectIdFromUrl(); @@ -82,10 +106,14 @@ export async function validateSamples({ projectId, body, }: { - projectId: number; - body: SampleRequest; -}) { - return await axios.post(`${URL}/${projectId}/samples/validate`, body); + projectId: string; + body: ValidateSampleNamesRequest; +}): Promise { + const response = await axios.post( + `${URL}/${projectId}/samples/validate`, + body + ); + return response.data; } export async function createSample({ diff --git a/src/main/webapp/resources/js/components/ant.design/forms/BlockRadioInput.jsx b/src/main/webapp/resources/js/components/ant.design/forms/BlockRadioInput.tsx similarity index 76% rename from src/main/webapp/resources/js/components/ant.design/forms/BlockRadioInput.jsx rename to src/main/webapp/resources/js/components/ant.design/forms/BlockRadioInput.tsx index f863c9f4bcf..baac02d1443 100644 --- a/src/main/webapp/resources/js/components/ant.design/forms/BlockRadioInput.jsx +++ b/src/main/webapp/resources/js/components/ant.design/forms/BlockRadioInput.tsx @@ -19,6 +19,11 @@ const RadioItem = styled.button` } `; +interface BlockRadioInputProps + extends React.ButtonHTMLAttributes { + children?: React.ReactNode; +} + /** * React component to Render a Ant Design Radio button in block. * @@ -27,6 +32,9 @@ const RadioItem = styled.button` * @returns {JSX.Element} * @constructor */ -export function BlockRadioInput({ children, ...props }) { +export function BlockRadioInput({ + children, + ...props +}: BlockRadioInputProps): JSX.Element { return {children}; } diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx similarity index 69% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 04dc67ff526..5933d3c2355 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -1,7 +1,7 @@ import React from "react"; import { useDispatch, useSelector } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; -import { Button, Radio, Typography } from "antd"; +import { Button, Radio, RadioChangeEvent, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { BlockRadioInput } from "../../../../components/ant.design/forms/BlockRadioInput"; import { @@ -9,6 +9,8 @@ import { IconArrowRight, } from "../../../../components/icons/Icons"; import { setSampleNameColumn } from "../services/importReducer"; +import { NavigateFunction } from "react-router/dist/lib/hooks"; +import { ImportDispatch, ImportState } from "../store"; const { Text } = Typography; @@ -18,15 +20,15 @@ const { Text } = Typography; * @returns {*} * @constructor */ -export function SampleMetadataImportMapHeaders() { - const { projectId } = useParams(); - const navigate = useNavigate(); - const [column, setColumn] = React.useState(); - const [loading, setLoading] = React.useState(false); +export function SampleMetadataImportMapHeaders(): JSX.Element { + const { projectId } = useParams<{ projectId: string }>(); + const navigate: NavigateFunction = useNavigate(); + const [column, setColumn] = React.useState(); + const [loading, setLoading] = React.useState(false); const { headers, sampleNameColumn } = useSelector( - (state) => state.importReducer + (state: ImportState) => state.importReducer ); - const dispatch = useDispatch(); + const dispatch: ImportDispatch = useDispatch(); React.useEffect(() => { if (!column) { @@ -35,9 +37,11 @@ export function SampleMetadataImportMapHeaders() { }, [sampleNameColumn, headers]); const onSubmit = async () => { - setLoading(true); - await dispatch(setSampleNameColumn({ projectId, column })); - navigate(`/${projectId}/sample-metadata/upload/review`); + if (projectId && column) { + setLoading(true); + await dispatch(setSampleNameColumn({ projectId, column })); + navigate(`/${projectId}/sample-metadata/upload/review`); + } }; return ( @@ -46,9 +50,9 @@ export function SampleMetadataImportMapHeaders() { setColumn(e.target.value)} + onChange={(e: RadioChangeEvent) => setColumn(e.target.value)} > - {headers.map((header, index) => ( + {headers.map((header: String, index: number) => ( (); + const navigate: NavigateFunction = useNavigate(); + const dispatch: ImportDispatch = useDispatch(); const [status, setStatus] = React.useState("process"); const [loading, setLoading] = React.useState(false); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.tsx index ba1c4679c06..9e8153c2251 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportWizard.tsx @@ -26,7 +26,7 @@ export function SampleMetadataImportWizard({ percent, children, }: SampleMetadataImportWizardProps): JSX.Element { - const { projectId } = useParams(); + const { projectId } = useParams<{ projectId: string }>(); return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts similarity index 64% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index 70f6d405344..849b06abebe 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -6,11 +6,42 @@ import { import { validateSampleName } from "../../../../apis/metadata/sample-utils"; import { createSample, + FieldUpdate, + MetadataItem, updateSample, + ValidateSampleNameModel, validateSamples, + ValidateSamplesResponse, } from "../../../../apis/projects/samples"; +import { ImportDispatch, ImportState } from "../store"; -const initialState = { +interface MetadataValidateDetailsItem { + isSampleNameValid: boolean; + foundSampleId?: number; +} + +interface MetadataSaveDetailsItem { + saved: boolean; + error?: string; +} +interface SaveMetadataResponse { + metadataSaveDetails: Record; +} + +interface SetSampleNameColumnResponse { + sampleNameColumn: string; + metadataValidateDetails: Record; +} + +export interface InitialState { + sampleNameColumn: string; + headers: string[]; + metadata: MetadataItem[]; + metadataValidateDetails: Record; + metadataSaveDetails: Record; +} + +const initialState: InitialState = { sampleNameColumn: "", headers: [], metadata: [], @@ -22,28 +53,33 @@ const initialState = { Redux async thunk for saving the metadata to samples. For more information on redux async thunks see: https://redux-toolkit.js.org/api/createAsyncThunk */ -export const saveMetadata = createAsyncThunk( +export const saveMetadata = createAsyncThunk< + SaveMetadataResponse, + { projectId: number; selectedMetadataKeys: string[] }, + { dispatch: ImportDispatch; state: ImportState } +>( `importReducer/saveMetadata`, async ({ projectId, selectedMetadataKeys }, { dispatch, getState }) => { - const state = getState(); - const sampleNameColumn = state.importReducer.sampleNameColumn; - const headers = state.importReducer.headers; - const metadata = state.importReducer.metadata; - const metadataValidateDetails = state.importReducer.metadataValidateDetails; - const metadataSaveDetails = {}; + const state: ImportState = getState(); + const sampleNameColumn: string = state.importReducer.sampleNameColumn; + const headers: string[] = state.importReducer.headers; + const metadata: MetadataItem[] = state.importReducer.metadata; + const metadataValidateDetails: Record = + state.importReducer.metadataValidateDetails; + const metadataSaveDetails: Record = {}; - const chunkSize = 100; + const chunkSize: number = 100; for (let i = 0; i < metadata.length; i = i + chunkSize) { - const promises = []; + const promises: Promise[] = []; for (let j = i; j < i + chunkSize && j < metadata.length; j++) { - const metadataItem = metadata[j]; - const index = metadataItem.rowKey; + const metadataItem: MetadataItem = metadata[j]; + const index: string = metadataItem.rowKey; if ( selectedMetadataKeys.includes(index) && metadataSaveDetails[index]?.saved !== true ) { - const name = metadataItem[sampleNameColumn]; - const metadataFields = Object.entries(metadataItem) + const name: string = metadataItem[sampleNameColumn]; + const metadataFields: FieldUpdate[] = Object.entries(metadataItem) .filter( ([key, value]) => headers.includes(key) && key !== sampleNameColumn @@ -108,30 +144,38 @@ export const saveMetadata = createAsyncThunk( Redux async thunk for setting the sample name column and enriching the metadata. For more information on redux async thunks see: https://redux-toolkit.js.org/api/createAsyncThunk */ -export const setSampleNameColumn = createAsyncThunk( +export const setSampleNameColumn = createAsyncThunk< + SetSampleNameColumnResponse, + { projectId: string; column: string }, + { state: ImportState } +>( `importReducer/setSampleNameColumn`, async ({ projectId, column }, { getState }) => { - const state = getState(); + const state: ImportState = getState(); const metadata = state.importReducer.metadata; - const metadataValidateDetails = {}; - const response = await validateSamples({ + const metadataValidateDetails: Record = + {}; + const samples: ValidateSampleNameModel[] = metadata + .filter((row) => row[column]) + .map((row) => ({ + name: row[column], + })); + const response: ValidateSamplesResponse = await validateSamples({ projectId: projectId, body: { - samples: metadata - .filter((row) => row[column]) - .map((row) => ({ - name: row[column], - })), + samples: samples, }, }); for (let i = 0; i < metadata.length; i++) { - const metadataItem = metadata[i]; - const index = metadataItem.rowKey; - const foundSample = response.data.samples.find( - (sample) => metadataItem[column] === sample.name - ); + const metadataItem: MetadataItem = metadata[i]; + const index: string = metadataItem.rowKey; + const sampleName: string = metadataItem[column]; + const foundSample: ValidateSampleNameModel | undefined = + response.samples.find( + (sample: ValidateSampleNameModel) => sampleName === sample.name + ); metadataValidateDetails[index] = { - isSampleNameValid: validateSampleName(metadataItem[column]), + isSampleNameValid: validateSampleName(sampleName), foundSampleId: foundSample?.ids?.at(0), }; } @@ -149,7 +193,7 @@ For more information on redux actions see: https://redux-toolkit.js.org/api/crea */ export const setHeaders = createAction( `importReducer/setHeaders`, - (headers) => ({ + (headers: string[]) => ({ payload: { headers }, }) ); @@ -160,7 +204,7 @@ For more information on redux actions see: https://redux-toolkit.js.org/api/crea */ export const setMetadata = createAction( `importReducer/setMetadata`, - (metadata) => ({ + (metadata: MetadataItem[]) => ({ payload: { metadata: metadata.map((metadataItem, index) => { return { @@ -178,7 +222,7 @@ For more information on redux actions see: https://redux-toolkit.js.org/api/crea */ export const setMetadataSaveDetails = createAction( `importReducer/setMetadataSaveDetails`, - (metadataSaveDetails) => ({ + (metadataSaveDetails: Record) => ({ payload: { metadataSaveDetails }, }) ); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts similarity index 68% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts index ff02574cfb6..5438708cde3 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts @@ -5,9 +5,14 @@ import { importReducer } from "./services/importReducer"; Redux Store for sample metadata importer. For more information on redux stores see: https://redux.js.org/tutorials/fundamentals/part-4-store */ -export default configureStore({ +const store = configureStore({ reducer: { importReducer, }, middleware: (getDefaultMiddleware) => getDefaultMiddleware(), }); + +export type ImportState = ReturnType; +export type ImportDispatch = typeof store.dispatch; + +export default store; From ce16ddf961fe2bbe146da655a60f80060471b5aa Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 5 Oct 2022 10:20:25 -0500 Subject: [PATCH 273/655] converting SampleMetadataImportReview.tsx to typescript --- .../resources/js/apis/projects/samples.ts | 4 +- .../SampleMetadataImportMapHeaders.tsx | 12 +- ...iew.jsx => SampleMetadataImportReview.tsx} | 114 ++++++++++-------- .../SampleMetadataImportUploadFile.tsx | 5 +- .../services/importReducer.ts | 2 +- .../projects/samples-metadata-import/store.ts | 3 + 6 files changed, 82 insertions(+), 58 deletions(-) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportReview.jsx => SampleMetadataImportReview.tsx} (66%) diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index 3faa6292af8..6a6ad1c7df2 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -120,7 +120,7 @@ export async function createSample({ projectId, body, }: { - projectId: number; + projectId: string; body: SampleRequest; }) { return await axios.post(`${URL}/${projectId}/samples/add-sample`, body); @@ -131,7 +131,7 @@ export async function updateSample({ sampleId, body, }: { - projectId: number; + projectId: string; sampleId: number; body: SampleRequest; }) { diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 5933d3c2355..f3ed142360e 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -1,5 +1,4 @@ import React from "react"; -import { useDispatch, useSelector } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; import { Button, Radio, RadioChangeEvent, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; @@ -10,7 +9,12 @@ import { } from "../../../../components/icons/Icons"; import { setSampleNameColumn } from "../services/importReducer"; import { NavigateFunction } from "react-router/dist/lib/hooks"; -import { ImportDispatch, ImportState } from "../store"; +import { + ImportDispatch, + ImportState, + useImportDispatch, + useImportSelector, +} from "../store"; const { Text } = Typography; @@ -25,10 +29,10 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { const navigate: NavigateFunction = useNavigate(); const [column, setColumn] = React.useState(); const [loading, setLoading] = React.useState(false); - const { headers, sampleNameColumn } = useSelector( + const { headers, sampleNameColumn } = useImportSelector( (state: ImportState) => state.importReducer ); - const dispatch: ImportDispatch = useDispatch(); + const dispatch: ImportDispatch = useImportDispatch(); React.useEffect(() => { if (!column) { diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx similarity index 66% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx index 89c929d04b2..c02b3c3e8e5 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx @@ -1,6 +1,14 @@ import React from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { Alert, Button, Table, Tag, Tooltip, Typography } from "antd"; +import { + Alert, + Button, + Table, + TableProps, + Tag, + Tooltip, + Typography, +} from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { IconArrowLeft, @@ -9,9 +17,18 @@ import { } from "../../../../components/icons/Icons"; import { red1, red2, red5 } from "../../../../styles/colors"; import styled from "styled-components"; -import { useDispatch, useSelector } from "react-redux"; import { saveMetadata } from "../services/importReducer"; import { getPaginationOptions } from "../../../../utilities/antdesign-table-utilities"; +import { NavigateFunction } from "react-router/dist/lib/hooks"; +import { + ImportDispatch, + ImportState, + useImportDispatch, + useImportSelector, +} from "../store"; +import { MetadataItem } from "../../../../apis/projects/samples"; +import { ColumnsType, ColumnType } from "antd/es/table"; +import { TableRowSelection } from "antd/lib/table/interface"; const { Paragraph, Text } = Typography; @@ -42,24 +59,24 @@ const MetadataTable = styled(Table)` * @returns {*} * @constructor */ -export function SampleMetadataImportReview() { - const { projectId } = useParams(); - const navigate = useNavigate(); - const [columns, setColumns] = React.useState([]); - const [selected, setSelected] = React.useState([]); - const [valid, setValid] = React.useState(true); - const [progress, setProgress] = React.useState(0); - const [loading, setLoading] = React.useState(false); +export function SampleMetadataImportReview(): JSX.Element { + const { projectId } = useParams<{ projectId: string }>(); + const navigate: NavigateFunction = useNavigate(); + const [columns, setColumns] = React.useState>([]); + const [selected, setSelected] = React.useState([]); + const [valid, setValid] = React.useState(true); + const [progress, setProgress] = React.useState(0); + const [loading, setLoading] = React.useState(false); const { headers, sampleNameColumn, metadata, metadataValidateDetails, metadataSaveDetails, - } = useSelector((state) => state.importReducer); - const dispatch = useDispatch(); + } = useImportSelector((state: ImportState) => state.importReducer); + const dispatch: ImportDispatch = useImportDispatch(); - const tagColumn = { + const tagColumn: ColumnType = { title: "", dataIndex: "tags", className: "t-metadata-uploader-new-column", @@ -72,6 +89,7 @@ export function SampleMetadataImportReview() { {i18n("SampleMetadataImportReview.table.filter.new")} ); + return text; }, filters: [ { @@ -85,17 +103,17 @@ export function SampleMetadataImportReview() { ], onFilter: (value, record) => value === "new" - ? !metadataValidateDetails[record.rowKey].foundSampleId - : metadataValidateDetails[record.rowKey].foundSampleId, + ? metadataValidateDetails[record.rowKey].foundSampleId !== undefined + : metadataValidateDetails[record.rowKey].foundSampleId === undefined, }; - const rowSelection = { + const rowSelection: TableRowSelection = { fixed: true, selectedRowKeys: selected, onChange: (selectedRowKeys) => { setSelected(selectedRowKeys); }, - getCheckboxProps: (record) => ({ + getCheckboxProps: (record: MetadataItem) => ({ disabled: !( metadataValidateDetails[record.rowKey].isSampleNameValid || metadataSaveDetails[record.rowKey]?.saved === true @@ -105,7 +123,7 @@ export function SampleMetadataImportReview() { React.useEffect(() => { const savedCount = Object.entries(metadataSaveDetails).filter( - ([key, metadataSaveDetailsItem]) => metadataSaveDetailsItem.saved + ([, metadataSaveDetailsItem]) => metadataSaveDetailsItem.saved ).length; setProgress((savedCount / selected.length) * 100); }, [metadataSaveDetails]); @@ -113,11 +131,11 @@ export function SampleMetadataImportReview() { React.useEffect(() => { setValid( !metadata.some( - (row) => metadataValidateDetails[row.rowKey].isSampleNameValid === false + (row) => !metadataValidateDetails[row.rowKey].isSampleNameValid ) ); - const sampleColumn = { + const sampleColumn: ColumnType = { title: sampleNameColumn, dataIndex: sampleNameColumn, fixed: "left", @@ -125,16 +143,15 @@ export function SampleMetadataImportReview() { onCell: (item) => { return { style: { - background: - metadataValidateDetails[item.rowKey].isSampleNameValid === true - ? null - : red1, + background: metadataValidateDetails[item.rowKey].isSampleNameValid + ? undefined + : red1, }, }; }, }; - const savedColumn = { + const savedColumn: ColumnType = { dataIndex: "saved", fixed: "left", width: 10, @@ -148,17 +165,18 @@ export function SampleMetadataImportReview() { ); + return text; }, }; - const otherColumns = headers + const otherColumns: ColumnsType = headers .filter((header) => header !== sampleNameColumn) .map((header) => ({ title: header, dataIndex: header, })); - const updatedColumns = [ + const updatedColumns: ColumnsType = [ savedColumn, sampleColumn, tagColumn, @@ -167,13 +185,13 @@ export function SampleMetadataImportReview() { setColumns(updatedColumns); setSelected( - metadata.map((row) => { - if ( - metadataValidateDetails[row.rowKey].isSampleNameValid || - metadataSaveDetails[row.rowKey]?.saved === true + metadata + .filter( + (row) => + metadataValidateDetails[row.rowKey].isSampleNameValid || + metadataSaveDetails[row.rowKey]?.saved === true ) - return row.rowKey; - }) + .map((row): string => row.rowKey) ); }, [metadataSaveDetails]); @@ -183,18 +201,20 @@ export function SampleMetadataImportReview() { .filter((metadataItem) => selected.includes(metadataItem.rowKey)) .map((metadataItem) => metadataItem.rowKey); - const response = await dispatch( - saveMetadata({ projectId, selectedMetadataKeys }) - ); + if (projectId) { + const response = await dispatch( + saveMetadata({ projectId, selectedMetadataKeys }) + ).unwrap(); - if ( - Object.entries(response.payload.metadataSaveDetails).filter( - ([, metadataSaveDetailsItem]) => metadataSaveDetailsItem.error - ).length === 0 - ) { - navigate(`/${projectId}/sample-metadata/upload/complete`); - } else { - setLoading(false); + if ( + Object.entries(response.metadataSaveDetails).filter( + ([, metadataSaveDetailsItem]) => metadataSaveDetailsItem.error + ).length === 0 + ) { + navigate(`/${projectId}/sample-metadata/upload/complete`); + } else { + setLoading(false); + } } }; @@ -218,13 +238,11 @@ export function SampleMetadataImportReview() { showIcon /> )} - ) => JSX.Element> className="t-metadata-uploader-review-table" rowKey={(row) => row.rowKey} rowClassName={(record) => - metadataSaveDetails[record.rowKey]?.saved === false - ? "row-error" - : null + metadataSaveDetails[record.rowKey]?.saved === false ? "row-error" : "" } rowSelection={rowSelection} columns={columns} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx index b9d64c09dd2..7065fe0ecd4 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx @@ -1,5 +1,4 @@ import React from "react"; -import { useDispatch } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; import { setHeaders, setMetadata } from "../services/importReducer"; import { notification, Spin, StepsProps, Typography, UploadProps } from "antd"; @@ -7,7 +6,7 @@ import { DragUpload } from "../../../../components/files/DragUpload"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import * as XLSX from "xlsx"; import { WorkBook } from "xlsx"; -import { ImportDispatch } from "../store"; +import { ImportDispatch, useImportDispatch } from "../store"; import { NavigateFunction } from "react-router/dist/lib/hooks"; const { Text } = Typography; @@ -21,7 +20,7 @@ const { Text } = Typography; export function SampleMetadataImportUploadFile(): JSX.Element { const { projectId } = useParams<{ projectId: string }>(); const navigate: NavigateFunction = useNavigate(); - const dispatch: ImportDispatch = useDispatch(); + const dispatch: ImportDispatch = useImportDispatch(); const [status, setStatus] = React.useState("process"); const [loading, setLoading] = React.useState(false); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index 849b06abebe..89a7b765a42 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -55,7 +55,7 @@ For more information on redux async thunks see: https://redux-toolkit.js.org/api */ export const saveMetadata = createAsyncThunk< SaveMetadataResponse, - { projectId: number; selectedMetadataKeys: string[] }, + { projectId: string; selectedMetadataKeys: string[] }, { dispatch: ImportDispatch; state: ImportState } >( `importReducer/saveMetadata`, diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts index 5438708cde3..61f63ddd955 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts @@ -1,5 +1,6 @@ import { configureStore } from "@reduxjs/toolkit"; import { importReducer } from "./services/importReducer"; +import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux"; /* Redux Store for sample metadata importer. @@ -14,5 +15,7 @@ const store = configureStore({ export type ImportState = ReturnType; export type ImportDispatch = typeof store.dispatch; +export const useImportDispatch: () => ImportDispatch = useDispatch; +export const useImportSelector: TypedUseSelectorHook = useSelector; export default store; From 997c1d55957a66a2872b42db2877973de50ce1ae Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 5 Oct 2022 10:33:31 -0500 Subject: [PATCH 274/655] converting SampleMetadataImportComplete.jsx to typescript --- ...lete.jsx => SampleMetadataImportComplete.tsx} | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportComplete.jsx => SampleMetadataImportComplete.tsx} (81%) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx similarity index 81% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx index 4d283f9e37b..cf2225a3379 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx @@ -3,7 +3,9 @@ import { useNavigate, useParams } from "react-router-dom"; import { Button, Result } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { setBaseUrl } from "../../../../utilities/url-utilities"; -import { useSelector } from "react-redux"; +import { MetadataItem } from "../../../../apis/projects/samples"; +import { ImportState, useImportSelector } from "../store"; +import { NavigateFunction } from "react-router/dist/lib/hooks"; /** * React component that displays Step #4 of the Sample Metadata Uploader. @@ -11,18 +13,18 @@ import { useSelector } from "react-redux"; * @returns {*} * @constructor */ -export function SampleMetadataImportComplete() { +export function SampleMetadataImportComplete(): JSX.Element { const { metadata, metadataValidateDetails, metadataSaveDetails } = - useSelector((state) => state.importReducer); + useImportSelector((state: ImportState) => state.importReducer); const samplesUpdatedCount = metadata.filter( - (metadataItem) => + (metadataItem: MetadataItem) => metadataSaveDetails[metadataItem.rowKey]?.saved === true && metadataValidateDetails[metadataItem.rowKey].foundSampleId ).length; const samplesCreatedCount = metadata.filter( - (metadataItem) => + (metadataItem: MetadataItem) => metadataSaveDetails[metadataItem.rowKey]?.saved === true && !metadataValidateDetails[metadataItem.rowKey].foundSampleId ).length; @@ -48,8 +50,8 @@ export function SampleMetadataImportComplete() { samplesCreatedCount ); - const { projectId } = useParams(); - const navigate = useNavigate(); + const { projectId } = useParams<{ projectId: string }>(); + const navigate: NavigateFunction = useNavigate(); React.useEffect(() => { setTimeout(() => { From 02987a9d5117bc73ae69d6b8f0289335cc6d0670 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 5 Oct 2022 11:01:22 -0500 Subject: [PATCH 275/655] more typescript --- .../js/apis/metadata/{sample-utils.js => sample-utils.ts} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename src/main/webapp/resources/js/apis/metadata/{sample-utils.js => sample-utils.ts} (62%) diff --git a/src/main/webapp/resources/js/apis/metadata/sample-utils.js b/src/main/webapp/resources/js/apis/metadata/sample-utils.ts similarity index 62% rename from src/main/webapp/resources/js/apis/metadata/sample-utils.js rename to src/main/webapp/resources/js/apis/metadata/sample-utils.ts index b009bdbac0a..c90e28b2bb8 100644 --- a/src/main/webapp/resources/js/apis/metadata/sample-utils.js +++ b/src/main/webapp/resources/js/apis/metadata/sample-utils.ts @@ -6,10 +6,10 @@ export const sampleNameRegex = new RegExp("^[A-Za-z0-9-_]{3,}$"); /** * Checks is the sample name is valid - * @param {string} sampleName - * @returns {boolean} + * @param sampleName the name of the sample + * @returns if the sample name is valid */ -export function validateSampleName(sampleName) { +export function validateSampleName(sampleName: string): boolean { if (sampleName) { return sampleNameRegex.test(sampleName); } else { From b90705220f76c7341fa47696e7bf24c0a07d6545 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 5 Oct 2022 14:55:56 -0500 Subject: [PATCH 276/655] getting around deadlock --- .../web/services/UIProjectSampleService.java | 2 +- .../irida/service/ProjectService.java | 11 +++++ .../service/impl/ProjectServiceImpl.java | 41 ++++++++++++------- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index e8f009e24b0..9e770582508 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -140,7 +140,7 @@ public ResponseEntity createSample(CreateSampleRequest request, Lo if (!Strings.isNullOrEmpty(request.getDescription())) { sample.setDescription(request.getDescription()); } - Join join = projectService.addSampleToProject(project, sample, true); + Join join = projectService.addSampleToProjectWithoutEvent(project, sample, true); if (request.getMetadata() != null) { Set metadataEntrySet = request.getMetadata().stream().map(entry -> { MetadataTemplateField field = metadataTemplateService.saveMetadataField( diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/ProjectService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/ProjectService.java index 0175f10d737..345dd7b9861 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/ProjectService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/ProjectService.java @@ -137,6 +137,17 @@ public Join updateUserGroupProjectMetadataRole(Project proje */ public Join addSampleToProject(Project project, Sample sample, boolean owner); + /** + * Add the specified {@link Sample} to the {@link Project} without creating an event. + * + * @param project the {@link Project} to add the {@link Sample} to. + * @param sample the {@link Sample} to add to the {@link Project}. If the {@link Sample} has not previously been + * persisted, the service will persist the {@link Sample}. + * @param owner Whether the project will have modification access for this sample + * @return a reference to the relationship resource created between the two entities. + */ + public Join addSampleToProjectWithoutEvent(Project project, Sample sample, boolean owner); + /** * Move a {@link Sample} from one {@link Project} to another * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java index 99a984ff0e4..568a1014753 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/impl/ProjectServiceImpl.java @@ -23,7 +23,6 @@ import org.springframework.security.access.prepost.PostFilter; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.security.core.userdetails.UserDetails; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; @@ -411,8 +410,9 @@ public ProjectSampleJoin addSampleToProject(Project project, Sample sample, bool // Check to ensure a sample with this sample name doesn't exist in this // project already if (sampleRepository.getSampleBySampleName(project, sample.getSampleName()) != null) { - throw new ExistingSampleNameException("Sample with the name '" + sample.getSampleName() - + "' already exists in project " + project.getId(), sample); + throw new ExistingSampleNameException( + "Sample with the name '" + sample.getSampleName() + "' already exists in project " + + project.getId(), sample); } // the sample hasn't been persisted before, persist it before calling @@ -438,13 +438,24 @@ public ProjectSampleJoin addSampleToProject(Project project, Sample sample, bool } } + /** + * {@inheritDoc} + */ + @Override + @Transactional + @PreAuthorize("hasAnyRole('ROLE_ADMIN', 'ROLE_SEQUENCER') or (hasPermission(#project, 'isProjectOwner'))") + public ProjectSampleJoin addSampleToProjectWithoutEvent(Project project, Sample sample, boolean owner) { + return addSampleToProject(project, sample, owner); + } + /** * {@inheritDoc} */ @Override @Transactional @LaunchesProjectEvent(SampleAddedProjectEvent.class) - @PreAuthorize("hasRole('ROLE_ADMIN') or ( hasPermission(#source, 'isProjectOwner') and hasPermission(#destination, 'isProjectOwner'))") + @PreAuthorize( + "hasRole('ROLE_ADMIN') or ( hasPermission(#source, 'isProjectOwner') and hasPermission(#destination, 'isProjectOwner'))") public ProjectSampleJoin moveSampleBetweenProjects(Project source, Project destination, Sample sample) { //read the existing ProjectSampleJoin to see if we're the owner ProjectSampleJoin projectSampleJoin = psjRepository.readSampleForProject(source, sample); @@ -608,15 +619,16 @@ public Collection getUserGroupProjectJoins(User user, Proj * {@inheritDoc} */ @Override - @PreAuthorize("hasRole('ROLE_ADMIN') or hasPermission(#subject,'isProjectOwner') and hasPermission(#relatedProject,'canReadProject')") + @PreAuthorize( + "hasRole('ROLE_ADMIN') or hasPermission(#subject,'isProjectOwner') and hasPermission(#relatedProject,'canReadProject')") public RelatedProjectJoin addRelatedProject(Project subject, Project relatedProject) { if (subject.equals(relatedProject)) { throw new IllegalArgumentException("Project cannot be related to itself"); } try { - RelatedProjectJoin relation = relatedProjectRepository - .save(new RelatedProjectJoin(subject, relatedProject)); + RelatedProjectJoin relation = relatedProjectRepository.save( + new RelatedProjectJoin(subject, relatedProject)); return relation; } catch (DataIntegrityViolationException e) { throw new EntityExistsException( @@ -692,8 +704,8 @@ public Join addReferenceFileToProject(Project project, R @Override @PreAuthorize("hasRole('ROLE_ADMIN') or hasPermission(#project, 'isProjectOwner')") public void removeReferenceFileFromProject(Project project, ReferenceFile file) { - List> referenceFilesForProject = prfjRepository - .findReferenceFilesForProject(project); + List> referenceFilesForProject = prfjRepository.findReferenceFilesForProject( + project); Join specificJoin = null; for (Join join : referenceFilesForProject) { if (join.getObject().equals(file)) { @@ -704,8 +716,9 @@ public void removeReferenceFileFromProject(Project project, ReferenceFile file) if (specificJoin != null) { prfjRepository.delete((ProjectReferenceFileJoin) specificJoin); } else { - throw new EntityNotFoundException("Cannot find a join for project [" + project.getName() - + "] and reference file [" + file.getLabel() + "]."); + throw new EntityNotFoundException( + "Cannot find a join for project [" + project.getName() + "] and reference file [" + file.getLabel() + + "]."); } } @@ -849,8 +862,8 @@ public Project createProjectWithSamples(Project project, List sampleIds, b @PostFilter("hasPermission(filterObject, 'canReadProject')") @Override public List getProjectsUsedInAnalysisSubmission(AnalysisSubmission submission) { - Set findSequencingObjectsForAnalysisSubmission = sequencingObjectRepository - .findSequencingObjectsForAnalysisSubmission(submission); + Set findSequencingObjectsForAnalysisSubmission = sequencingObjectRepository.findSequencingObjectsForAnalysisSubmission( + submission); // get available projects Set projectsInAnalysis = getProjectsForSequencingObjects(findSequencingObjectsForAnalysisSubmission); @@ -891,7 +904,7 @@ private static final Sort getOrDefaultSort(Sort sort) { * @param projectRole The {@link ProjectRole} to search for. * @param user The user to search * @return a {@link Specification} to search for {@link Project} where the specified {@link User} has a certain - * {@link ProjectRole}. + * {@link ProjectRole}. */ private static final Specification getProjectJoinsWithRole(User user, ProjectRole projectRole) { return new Specification() { From 3a17030dbc39fb683f734c7a037258b44a83df2f Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 5 Oct 2022 15:23:05 -0500 Subject: [PATCH 277/655] fix test --- .../irida/ria/unit/web/services/UIProjectSampleServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java index dd0fc9c0a46..c5aa1d4256d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java @@ -60,6 +60,8 @@ public void setUp() { Join join = new ProjectSampleJoin(PROJECT_1, sample, true); when(projectService.addSampleToProject(any(Project.class), any(Sample.class), any(Boolean.class))).thenReturn( join); + when(projectService.addSampleToProjectWithoutEvent(any(Project.class), any(Sample.class), + any(Boolean.class))).thenReturn(join); } @Test From a50d6d745b0f1bc4792cda3705ae915fbfe0ec5e Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 7 Oct 2022 10:42:29 -0500 Subject: [PATCH 278/655] eslinting --- .../SampleMetadataImportComplete.tsx | 2 +- .../SampleMetadataImportMapHeaders.tsx | 4 +- .../components/SampleMetadataImportReview.tsx | 72 ++++++++++--------- .../SampleMetadataImportUploadFile.tsx | 21 ++++-- .../services/importReducer.ts | 20 +++--- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx index cf2225a3379..6a05b9bba2c 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx @@ -57,7 +57,7 @@ export function SampleMetadataImportComplete(): JSX.Element { setTimeout(() => { navigate(`/${projectId}/sample-metadata/upload/file`); }, 10000); - }, []); + }, [navigate, projectId]); return ( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index f3ed142360e..ed7e68d1235 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -38,7 +38,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { if (!column) { setColumn(sampleNameColumn ? sampleNameColumn : headers[0]); } - }, [sampleNameColumn, headers]); + }, [sampleNameColumn, headers, column]); const onSubmit = async () => { if (projectId && column) { @@ -56,7 +56,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { value={column} onChange={(e: RadioChangeEvent) => setColumn(e.target.value)} > - {headers.map((header: String, index: number) => ( + {headers.map((header: string, index: number) => ( state.importReducer); const dispatch: ImportDispatch = useImportDispatch(); - const tagColumn: ColumnType = { - title: "", - dataIndex: "tags", - className: "t-metadata-uploader-new-column", - fixed: "left", - width: 70, - render: (text, item) => { - if (!metadataValidateDetails[item.rowKey].foundSampleId) - return ( - - {i18n("SampleMetadataImportReview.table.filter.new")} - - ); - return text; - }, - filters: [ - { - text: i18n("SampleMetadataImportReview.table.filter.new"), - value: "new", - }, - { - text: i18n("SampleMetadataImportReview.table.filter.existing"), - value: "existing", - }, - ], - onFilter: (value, record) => - value === "new" - ? metadataValidateDetails[record.rowKey].foundSampleId !== undefined - : metadataValidateDetails[record.rowKey].foundSampleId === undefined, - }; - const rowSelection: TableRowSelection = { fixed: true, selectedRowKeys: selected, @@ -126,7 +95,7 @@ export function SampleMetadataImportReview(): JSX.Element { ([, metadataSaveDetailsItem]) => metadataSaveDetailsItem.saved ).length; setProgress((savedCount / selected.length) * 100); - }, [metadataSaveDetails]); + }, [metadataSaveDetails, selected.length]); React.useEffect(() => { setValid( @@ -169,6 +138,37 @@ export function SampleMetadataImportReview(): JSX.Element { }, }; + const tagColumn: ColumnType = { + title: "", + dataIndex: "tags", + className: "t-metadata-uploader-new-column", + fixed: "left", + width: 70, + render: (text, item) => { + if (!metadataValidateDetails[item.rowKey].foundSampleId) + return ( + + {i18n("SampleMetadataImportReview.table.filter.new")} + + ); + return text; + }, + filters: [ + { + text: i18n("SampleMetadataImportReview.table.filter.new"), + value: "new", + }, + { + text: i18n("SampleMetadataImportReview.table.filter.existing"), + value: "existing", + }, + ], + onFilter: (value, record) => + value === "new" + ? metadataValidateDetails[record.rowKey].foundSampleId !== undefined + : metadataValidateDetails[record.rowKey].foundSampleId === undefined, + }; + const otherColumns: ColumnsType = headers .filter((header) => header !== sampleNameColumn) .map((header) => ({ @@ -193,7 +193,13 @@ export function SampleMetadataImportReview(): JSX.Element { ) .map((row): string => row.rowKey) ); - }, [metadataSaveDetails]); + }, [ + headers, + metadata, + metadataSaveDetails, + metadataValidateDetails, + sampleNameColumn, + ]); const save = async () => { setLoading(true); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx index 7065fe0ecd4..6ed28fb336e 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx @@ -8,6 +8,8 @@ import * as XLSX from "xlsx"; import { WorkBook } from "xlsx"; import { ImportDispatch, useImportDispatch } from "../store"; import { NavigateFunction } from "react-router/dist/lib/hooks"; +import { MetadataItem } from "../../../../apis/projects/samples"; +import { RcFile, UploadFileStatus } from "antd/lib/upload/interface"; const { Text } = Typography; @@ -29,20 +31,25 @@ export function SampleMetadataImportUploadFile(): JSX.Element { showUploadList: false, accept: ".xls,.xlsx,.csv", onChange(info: { - file: { originFileObj?: any; name?: any; status?: any }; + file: { + originFileObj?: RcFile; + name?: string; + status?: UploadFileStatus; + }; }) { const { status } = info.file; if (status !== "uploading") { setLoading(true); - let reader = new FileReader(); + const reader = new FileReader(); if (reader.readAsBinaryString) { - reader.onload = (e) => { + reader.onload = () => { const workbook: WorkBook = XLSX.read(reader.result, { type: "binary", raw: true, }); - const firstSheet: string = workbook.SheetNames[0]; - const rows: any[] = XLSX.utils.sheet_to_json( + const { SheetNames } = workbook; + const [firstSheet] = SheetNames; + const rows: MetadataItem[] = XLSX.utils.sheet_to_json( workbook.Sheets[firstSheet], { rawNumbers: false, @@ -51,7 +58,9 @@ export function SampleMetadataImportUploadFile(): JSX.Element { dispatch(setHeaders(Object.keys(rows[0]))); dispatch(setMetadata(rows)); }; - reader.readAsBinaryString(info.file.originFileObj); + if (info.file.originFileObj) { + reader.readAsBinaryString(info.file.originFileObj); + } } } if (status === "done") { diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index 89a7b765a42..c538c65d451 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -61,16 +61,13 @@ export const saveMetadata = createAsyncThunk< `importReducer/saveMetadata`, async ({ projectId, selectedMetadataKeys }, { dispatch, getState }) => { const state: ImportState = getState(); - const sampleNameColumn: string = state.importReducer.sampleNameColumn; - const headers: string[] = state.importReducer.headers; - const metadata: MetadataItem[] = state.importReducer.metadata; - const metadataValidateDetails: Record = - state.importReducer.metadataValidateDetails; + const { sampleNameColumn, headers, metadata, metadataValidateDetails } = + state.importReducer; const metadataSaveDetails: Record = {}; - const chunkSize: number = 100; + const chunkSize = 100; for (let i = 0; i < metadata.length; i = i + chunkSize) { - const promises: Promise[] = []; + const promises: Promise[] = []; for (let j = i; j < i + chunkSize && j < metadata.length; j++) { const metadataItem: MetadataItem = metadata[j]; const index: string = metadataItem.rowKey; @@ -81,8 +78,7 @@ export const saveMetadata = createAsyncThunk< const name: string = metadataItem[sampleNameColumn]; const metadataFields: FieldUpdate[] = Object.entries(metadataItem) .filter( - ([key, value]) => - headers.includes(key) && key !== sampleNameColumn + ([key]) => headers.includes(key) && key !== sampleNameColumn ) .map(([key, value]) => ({ field: key, value })); const sampleId = metadataValidateDetails[index].foundSampleId; @@ -97,7 +93,7 @@ export const saveMetadata = createAsyncThunk< metadata: metadataFields, }, }) - .then((response) => { + .then(() => { metadataSaveDetails[index] = { saved: true }; }) .catch((error) => { @@ -116,7 +112,7 @@ export const saveMetadata = createAsyncThunk< metadata: metadataFields, }, }) - .then((response) => { + .then(() => { metadataSaveDetails[index] = { saved: true }; }) .catch((error) => { @@ -152,7 +148,7 @@ export const setSampleNameColumn = createAsyncThunk< `importReducer/setSampleNameColumn`, async ({ projectId, column }, { getState }) => { const state: ImportState = getState(); - const metadata = state.importReducer.metadata; + const { metadata } = state.importReducer; const metadataValidateDetails: Record = {}; const samples: ValidateSampleNameModel[] = metadata From a081bb6f3621532e3c0720e8c11b30c707ca1b1f Mon Sep 17 00:00:00 2001 From: Eric Enns Date: Fri, 7 Oct 2022 15:52:09 -0500 Subject: [PATCH 279/655] chore: update changelog for unreleased changes and future release. --- CHANGELOG.md | 3 +++ build.gradle.kts | 2 +- .../bioinformatics/irida/database/all-changes.xml | 1 + .../irida/database/changesets/unreleased/all-changes.xml | 4 ++++ 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f97c48f678..598fb8d144a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog +## [Unreleased] ## [22.09] - 2022/10/07 * [Developer/UI]: Refreshed the create new user account page. See [PR 1285](https://github.com/phac-nml/irida/pull/1285) * [Developer/UI]: Added in typescript support to webpack build, moving forward all new frontend development will use typescript. See [PR 1294](https://github.com/phac-nml/irida/pull/1294) for more. @@ -130,6 +131,8 @@ ## [...previous](https://github.com/phac-nml/irida/blob/21.09.2/CHANGELOG.md) +[Unreleased]: https://github.com/phac-nml/irida/compare/22.09...HEAD + [22.09]: https://github.com/phac-nml/irida/compare/22.05.5...22.09 [22.05.5]: https://github.com/phac-nml/irida/compare/22.05.4...22.05.5 [22.05.4]: https://github.com/phac-nml/irida/compare/22.05.3...22.05.4 diff --git a/build.gradle.kts b/build.gradle.kts index 293924106f0..9e81cfbe510 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,7 +14,7 @@ plugins { } group = "ca.corefacility.bioinformatics" -version = "22.09" +version = "23.01-SNAPSHOT" description = "irida" java { diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml index ce3041fa9a9..756a961cd9f 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/all-changes.xml @@ -72,4 +72,5 @@ + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml new file mode 100644 index 00000000000..61d30a2d3fa --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml @@ -0,0 +1,4 @@ + + + \ No newline at end of file From 8d2cb8705793dd71112a3070f414a508971da94a Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Fri, 7 Oct 2022 16:16:01 -0500 Subject: [PATCH 280/655] Merged dev and fixed merge conflicts --- .../irida/config/web/IridaUIWebConfig.java | 2 +- .../web/analysis/AnalysisAjaxController.java | 4 ++-- .../js/contexts/AnalysisOutputsContext.js | 15 ++++++++---- .../analysis/components/AnalysisSistr.jsx | 7 +++--- .../components/settings/AnalysisShare.jsx | 24 ++++++++++--------- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java index 3c78831499a..f384bc18281 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java @@ -272,7 +272,7 @@ public void addArgumentResolvers(List argumentRes */ private Set additionalDialects() { Set dialects = new HashSet<>(); - dialects.add(new WebpackerDialect(!env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); + dialects.add(new WebpackerDialect(env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); dialects.add(new SpringSecurityDialect()); dialects.add(new LayoutDialect()); dialects.add(new DataAttributeDialect()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java index c2572de39ea..6ee2faed342 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/analysis/AnalysisAjaxController.java @@ -430,7 +430,7 @@ private void addFirstLine(AnalysisOutputFileInfo info, AnalysisOutputFile aof) { } else { info.setFilePointer(0L); } - } catch (IOException e) { + } catch (StorageException | IOException e) { logger.error("Could not get file input stream '" + aofFile + "' " + e); } } @@ -728,7 +728,7 @@ public AnalysisSistrResults getSistrAnalysis(@PathVariable Long id) { } } catch (JsonParseException | JsonMappingException e) { logger.error("Error attempting to parse file [" + path + "] as JSON", e); - } catch (IOException e) { + } catch (StorageException | IOException e) { logger.error("Error reading file input stream [" + path + "]", e); } } else { diff --git a/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js b/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js index dee65de7b9e..3bedd9f78eb 100644 --- a/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js +++ b/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js @@ -12,7 +12,12 @@ import { getOutputInfo } from "../apis/analysis/analysis"; const initialContext = { outputs: null, fileTypes: [ - { hasJsonFile: false, hasTabularFile: false, hasTextFile: false, hasHtmlFile: false }, + { + hasJsonFile: false, + hasTabularFile: false, + hasTextFile: false, + hasHtmlFile: false, + }, ], }; @@ -25,9 +30,8 @@ const imageFileExtSet = new Set(["png", "jpeg", "jpg"]); const htmlFileExtSet = new Set(["html", "html-zip"]); function AnalysisOutputsProvider(props) { - const [analysisOutputsContext, setAnalysisOutputsContext] = useState( - initialContext - ); + const [analysisOutputsContext, setAnalysisOutputsContext] = + useState(initialContext); const { analysisIdentifier } = useContext(AnalysisContext); function getPreviewForFileType(fileExt, type) { @@ -62,6 +66,7 @@ function AnalysisOutputsProvider(props) { let hasHtmlFile = false; getOutputInfo(analysisIdentifier).then((data) => { + console.log(data); // Check if json, tab, and/or text files exist // Used by output file preview to only display // tabs that are required @@ -105,7 +110,7 @@ function AnalysisOutputsProvider(props) { hasTextFile, hasExcelFile, hasImageFile, - hasHtmlFile + hasHtmlFile, }, ], }; diff --git a/src/main/webapp/resources/js/pages/analysis/components/AnalysisSistr.jsx b/src/main/webapp/resources/js/pages/analysis/components/AnalysisSistr.jsx index 787dc3d039e..aeef9e61567 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/AnalysisSistr.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/AnalysisSistr.jsx @@ -30,14 +30,15 @@ export default function AnalysisSistr({ baseUrl }) { const { analysisIdentifier } = useContext(AnalysisContext); const [sistrResults, setSistrResults] = useState(null); - const DEFAULT_URL = - setBaseUrl(`/analysis/${analysisIdentifier}/` + ANALYSIS.SISTR); + const DEFAULT_URL = setBaseUrl( + `/analysis/${analysisIdentifier}/` + ANALYSIS.SISTR + ); const pathRegx = new RegExp(/([a-zA-Z_]+)$/); const keyname = location.pathname.match(pathRegx); // On load gets the SISTR results by causing the SistResult Object generation and conversion to Ajax Ojbect useEffect(() => { - getSistrResults(analysisIdentifier).then(data => { + getSistrResults(analysisIdentifier).then((data) => { setSistrResults(data); }); }, []); diff --git a/src/main/webapp/resources/js/pages/analysis/components/settings/AnalysisShare.jsx b/src/main/webapp/resources/js/pages/analysis/components/settings/AnalysisShare.jsx index 86607ead55f..46c7a192f38 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/settings/AnalysisShare.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/settings/AnalysisShare.jsx @@ -21,7 +21,8 @@ import { getSharedProjects, updateSharedProject, } from "../../../../apis/analysis/analysis"; -import { TabPanelContent } from "../../../../components/tabs/TabPanelContent"; + +import { TabPanelContent } from "../../../../components/tabs"; const { Title } = Typography; @@ -35,6 +36,7 @@ export default function AnalysisShare() { AnalysisDetailsContext ); const { analysisContext, analysisIdentifier } = useContext(AnalysisContext); + const { analysisShareContext, storeSharedProjects, @@ -158,16 +160,16 @@ export default function AnalysisShare() { ) : null} {analysisDetailsContext.canShareToSamples && - analysisDetailsContext.allowedToModifySample && - analysisContext.isCompleted ? ( -
    - {i18n("AnalysisShare.saveResults")} - {renderSaveToRelatedSamples()} -
    - ) : null} + analysisDetailsContext.allowedToModifySample && + analysisContext.isCompleted && ( +
    + {i18n("AnalysisShare.saveResults")} + {renderSaveToRelatedSamples()} +
    + )} ); } From d5b971a36841280535d5dd247ce0c8085ca98994 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Mon, 10 Oct 2022 20:11:57 -0500 Subject: [PATCH 281/655] removing todo --- .../irida/ria/web/services/UIProjectSampleService.java | 8 ++++++-- .../samples-metadata-import/services/importReducer.ts | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index 9e770582508..5aa74b27b2d 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -169,8 +169,12 @@ public ResponseEntity updateSample(UpdateSampleRequest request, Lo try { Sample sample = sampleService.read(sampleId); sample.setSampleName(request.getName()); - sample.setOrganism(request.getOrganism()); - sample.setDescription(request.getDescription()); + if (!Strings.isNullOrEmpty(request.getOrganism())) { + sample.setOrganism(request.getOrganism()); + } + if (request.getDescription() != null) { + sample.setDescription(request.getDescription()); + } if (request.getMetadata() != null) { Set metadataEntrySet = request.getMetadata().stream().map(entry -> { MetadataTemplateField field = metadataTemplateService.saveMetadataField( diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index c538c65d451..89113490c05 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -89,7 +89,6 @@ export const saveMetadata = createAsyncThunk< sampleId, body: { name, - // TODO: Don't overwrite organism & description metadata: metadataFields, }, }) From 94700e8fa83a373382d2e5c6f0741f91ff405a2f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 10 Oct 2022 20:36:17 -0500 Subject: [PATCH 282/655] Updated AnalysisBioHansel logic. Updated db files --- .../irida/config/web/IridaUIWebConfig.java | 2 +- .../updater/impl/BioHanselSampleUpdater.java | 3 +++ .../database/changesets/22.09/all-changes.xml | 4 +++- .../changesets/unreleased/all-changes.xml | 8 +------ .../BioHansel/2.0.0/messages_en.properties | 3 ++- .../js/contexts/AnalysisOutputsContext.js | 1 - .../analysis/components/AnalysisBioHansel.jsx | 22 ++++++++++++++++--- 7 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java index f384bc18281..3c78831499a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java @@ -272,7 +272,7 @@ public void addArgumentResolvers(List argumentRes */ private Set additionalDialects() { Set dialects = new HashSet<>(); - dialects.add(new WebpackerDialect(env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); + dialects.add(new WebpackerDialect(!env.acceptsProfiles(Profiles.of(SPRING_PROFILE_PRODUCTION)))); dialects.add(new SpringSecurityDialect()); dialects.add(new LayoutDialect()); dialects.add(new DataAttributeDialect()); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java index 34849ceef8e..376c5e77ade 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/pipeline/results/updater/impl/BioHanselSampleUpdater.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Component; import ca.corefacility.bioinformatics.irida.exceptions.PostProcessingException; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplate; import ca.corefacility.bioinformatics.irida.model.sample.Sample; @@ -113,6 +114,8 @@ public void update(Collection samples, AnalysisSubmission analysis) thro } else { throw new PostProcessingException(filePath + " not correctly formatted. Expected valid JSON."); } + } catch (StorageException e) { + throw new PostProcessingException("Unable to read file input stream ", e); } catch (IOException e) { throw new PostProcessingException("Error parsing JSON from " + filePath, e); } diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.09/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.09/all-changes.xml index 76e17ee933b..4167515f7de 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.09/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/22.09/all-changes.xml @@ -1,6 +1,8 @@ - diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml index 4bb2e10504e..1574d30e37a 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml @@ -2,11 +2,5 @@ - - - - - - - + diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/model/workflow/analysis/type/workflows/BioHansel/2.0.0/messages_en.properties b/src/main/resources/ca/corefacility/bioinformatics/irida/model/workflow/analysis/type/workflows/BioHansel/2.0.0/messages_en.properties index 21ccacd2fb1..1036681cd2d 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/model/workflow/analysis/type/workflows/BioHansel/2.0.0/messages_en.properties +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/model/workflow/analysis/type/workflows/BioHansel/2.0.0/messages_en.properties @@ -23,7 +23,7 @@ pipeline.parameters.bio_hansel.bio_hansel-1-qc_vals.min_ambiguous_tiles=QC: Mini pipeline.parameters.bio_hansel.bio_hansel-1-qc_vals.max_intermediate_tiles=QC: Maximum proportion of subtype-specific tiles that can be missing before a QC warning of a possible intermediate subtype (Numeric; 0.0 to 1.0) # ========================================================================================== # -# ANALYSIS DETAILS BIO HANSEL COMPONENT # +# ANALYSIS DETAILS BIO HANSEL COMPONENT # # ========================================================================================== # AnalysisBioHansel.sampleName=Sample Name AnalysisBioHansel.bioHansel=Bio Hansel @@ -34,5 +34,6 @@ AnalysisBioHansel.averageTileFrequency=Average Tile Frequency AnalysisBioHansel.qualityControlStatus=Quality Control Status AnalysisBioHansel.bioHanselInformation=Bio Hansel Information AnalysisBioHansel.resultsUnavailable=Bio Hansel results are unavailable +AnalysisBioHansel.loadingResults=Loading Bio Hansel results workflow.label.share-analysis-samples.BIO_HANSEL=Save bio_hansel results to Project Line List Metadata diff --git a/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js b/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js index 3bedd9f78eb..ec5007284c9 100644 --- a/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js +++ b/src/main/webapp/resources/js/contexts/AnalysisOutputsContext.js @@ -66,7 +66,6 @@ function AnalysisOutputsProvider(props) { let hasHtmlFile = false; getOutputInfo(analysisIdentifier).then((data) => { - console.log(data); // Check if json, tab, and/or text files exist // Used by output file preview to only display // tabs that are required diff --git a/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx b/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx index e20214af149..e8ef0dd16cf 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx @@ -12,6 +12,8 @@ import { Error, Success } from "../../../components/icons"; import { Warning } from "../../../components/icons/Warning"; import { BasicList } from "../../../components/lists"; import ScrollableSection from "./ScrollableSection"; +import { WarningAlert } from "../../../components/alerts"; +import { ContentLoading } from "../../../components/loader"; export default function AnalysisBioHansel() { const { analysisIdentifier } = useContext(AnalysisContext); @@ -19,6 +21,7 @@ export default function AnalysisBioHansel() { AnalysisOutputsContext ); const [bioHanselResults, setBioHanselResults] = useState(null); + const [loading, setLoading] = useState(true); // On load gets the bio hansel results. If the file is not found then set to undefined useEffect(() => { @@ -29,6 +32,7 @@ export default function AnalysisBioHansel() { useEffect(() => { getBioHanselResults(); + setLoading(false); }, [analysisOutputsContext.outputs]); function getBioHanselResults() { @@ -130,9 +134,21 @@ export default function AnalysisBioHansel() { return ( - - - + {!loading ? ( + bioHanselResults !== null && bioHanselResults !== undefined ? ( + + + + ) : ( + + ) + ) : ( + + )} ); } From 1f48f877a343b8052c1700aec294b79fd821c237 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 11 Oct 2022 14:09:43 -0500 Subject: [PATCH 283/655] Updated sample concatenation UI to better support object storage (loading indicatior, error alert). Updated biohansel component to not flash warning alert before results are displayed --- .../impl/SequenceFilePairConcatenator.java | 3 + .../IridaFileStorageLocalUtilityImpl.java | 2 +- .../web/samples/SamplesAjaxController.java | 12 ++-- .../samples/dto/SampleConcatenationModel.java | 26 ++++++++ .../ria/web/services/UISampleService.java | 10 +++- src/main/resources/i18n/messages.properties | 3 + .../resources/js/apis/samples/samples.ts | 5 ++ .../alerts/{ErrorAlert.jsx => ErrorAlert.tsx} | 19 +++++- .../components/SampleFileContenate.tsx | 59 ++++++++++++++++--- .../analysis/components/AnalysisBioHansel.jsx | 3 +- .../samples/SamplesAjaxControllerTest.java | 3 +- .../web/services/UISampleServiceTest.java | 8 +-- 12 files changed, 128 insertions(+), 25 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SampleConcatenationModel.java rename src/main/webapp/resources/js/components/alerts/{ErrorAlert.jsx => ErrorAlert.tsx} (63%) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java index ade211d4db0..f44860625ae 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/processing/concatenate/impl/SequenceFilePairConcatenator.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.processing.concatenate.impl; import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFilePair; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; @@ -70,6 +71,8 @@ public SequenceFilePair concatenateFiles(List toConc try { iridaFileStorageUtility.appendToFile(forwardFile, forwardSequenceFile); iridaFileStorageUtility.appendToFile(reverseFile, reverseSequenceFile); + } catch( StorageException e) { + throw new StorageException(e.getMessage()); } catch (IOException e) { throw new ConcatenateException("Could not append files", e); } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java index 4679684f6e3..6f8e413bad7 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java @@ -155,7 +155,7 @@ public void appendToFile(Path target, SequenceFile file) throws IOException { p += in.transferTo(p, l - p, out); } } catch (IOException e) { - throw new IOException("Could not open input file for reading", e); + throw new StorageException("Could not open input file for reading", e); } } catch (IOException e) { diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java index 5bc98e39c70..498a976b206 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/SamplesAjaxController.java @@ -11,6 +11,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.exceptions.EntityExistsException; import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.*; @@ -333,16 +334,17 @@ public void downloadAssembly(@PathVariable Long sampleId, @RequestParam Long gen * @return {@link ResponseEntity} with the new concatenated sequencing object */ @PostMapping(value = "/{sampleId}/files/concatenate") - public ResponseEntity> concatenateSequenceFiles(@PathVariable Long sampleId, + public ResponseEntity concatenateSequenceFiles(@PathVariable Long sampleId, @RequestParam(name = "sequencingObjectIds") Set sequenceObjectIds, @RequestParam(name = "newFileName") String newFileName, - @RequestParam(name = "removeOriginals", defaultValue = "false", required = false) boolean removeOriginals) { + @RequestParam(name = "removeOriginals", defaultValue = "false", required = false) boolean removeOriginals, + Locale locale) { try { return ResponseEntity.ok(uiSampleService.concatenateSequenceFiles(sampleId, sequenceObjectIds, newFileName, - removeOriginals)); - } catch (ConcatenateException e) { + removeOriginals,locale)); + } catch (ConcatenateException | StorageException e) { return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(null); + .body(new SampleConcatenationModel(e.getMessage())); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SampleConcatenationModel.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SampleConcatenationModel.java new file mode 100644 index 00000000000..f5b57b3a3cc --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/samples/dto/SampleConcatenationModel.java @@ -0,0 +1,26 @@ +package ca.corefacility.bioinformatics.irida.ria.web.samples.dto; + +import java.util.List; + +public class SampleConcatenationModel { + private List sampleSequencingObjectFileModels; + private String concatenationError; + + public SampleConcatenationModel(List sampleSequencingObjectFileModels) { + this.sampleSequencingObjectFileModels = sampleSequencingObjectFileModels; + this.concatenationError = null; + } + + public SampleConcatenationModel(String concatenationError) { + this.sampleSequencingObjectFileModels = null; + this.concatenationError = concatenationError; + } + + public List getSampleSequencingObjectFileModels() { + return sampleSequencingObjectFileModels; + } + + public String getConcatenationError() { + return concatenationError; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UISampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UISampleService.java index 62e325f7790..142331c5ffa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UISampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UISampleService.java @@ -22,6 +22,7 @@ import ca.corefacility.bioinformatics.irida.exceptions.ConcatenateException; import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException; +import ca.corefacility.bioinformatics.irida.exceptions.StorageException; import ca.corefacility.bioinformatics.irida.model.assembly.UploadedAssembly; import ca.corefacility.bioinformatics.irida.model.enums.ProjectMetadataRole; import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField; @@ -1093,8 +1094,8 @@ public List uploadAssemblies(Long sampleId, Multi * @return The concatenated sequencing object in a {@link SampleSequencingObjectFileModel} * @throws ConcatenateException if there was an error concatenating the files */ - public List concatenateSequenceFiles(Long sampleId, Set objectIds, - String filename, boolean removeOriginals) throws ConcatenateException { + public SampleConcatenationModel concatenateSequenceFiles(Long sampleId, Set objectIds, + String filename, boolean removeOriginals, Locale locale) throws ConcatenateException { Sample sample = sampleService.read(sampleId); List sampleSequencingObjectFileModels = new ArrayList<>(); Iterable readMultiple = sequencingObjectService.readMultiple(objectIds); @@ -1128,9 +1129,12 @@ public List concatenateSequenceFiles(Long sampl sampleSequencingObjectFileModels.add( new SampleSequencingObjectFileModel(sequencingObject, firstFileSize, secondFileSize, sequencingObject.getQcEntries())); - return sampleSequencingObjectFileModels; + return new SampleConcatenationModel(sampleSequencingObjectFileModels); } catch (ConcatenateException ex) { throw new ConcatenateException(ex.getMessage()); + } catch (StorageException e) { + throw new StorageException(messageSource.getMessage("server.SampleFilesConcatenate.error.reading.file", + new Object[] { }, locale)); } } diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index ef7f00c7677..d5bb2f02f60 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2287,6 +2287,9 @@ SampleFilesConcatenate.title=Concatenate the following sequencing objects SampleFilesConcatenate.newFileName=New File Name: SampleFilesConcatenate.removeOriginalFiles=Would you like to remove the original files? SampleFilesConcatenate.checkboxDescription=Select 2 or more sequencing objects of the same type to concatenate +SampleFilesConcatenate.nameLengthValidationError=File name must be atleast 3 characters + +server.SampleFilesConcatenate.error.reading.file=Unable to concatenate selected files. There was an error reading 1 or more of the files. # ========================================================================================== # # ADD SAMPLE REACT COMPONENT # diff --git a/src/main/webapp/resources/js/apis/samples/samples.ts b/src/main/webapp/resources/js/apis/samples/samples.ts index 20afdde711a..f7611602cf9 100644 --- a/src/main/webapp/resources/js/apis/samples/samples.ts +++ b/src/main/webapp/resources/js/apis/samples/samples.ts @@ -97,6 +97,11 @@ export interface SampleMetadataFieldEntry { metadataRestriction: ProjectMetadataRole; } +export interface SampleConcatenationObject { + sampleSequencingObjectFileModels: SampleSequencingObject[]; + concatenationError: string; +} + export interface SampleSequencingObject { automatedAssembly: AnalysisSubmission; fileInfo: SequencingObject; diff --git a/src/main/webapp/resources/js/components/alerts/ErrorAlert.jsx b/src/main/webapp/resources/js/components/alerts/ErrorAlert.tsx similarity index 63% rename from src/main/webapp/resources/js/components/alerts/ErrorAlert.jsx rename to src/main/webapp/resources/js/components/alerts/ErrorAlert.tsx index 83671ddacc7..4a93d4111e7 100644 --- a/src/main/webapp/resources/js/components/alerts/ErrorAlert.jsx +++ b/src/main/webapp/resources/js/components/alerts/ErrorAlert.tsx @@ -4,6 +4,17 @@ import React from "react"; import { Alert } from "antd"; +import { SampleFileConcatenateProps } from "../samples/components/SampleFileContenate"; + +export interface Dictionary { + [Key: string]: T; +} + +export interface ErrorAlertProps { + message: string; + description?: string; + props?: Dictionary; +} /** * Stateless UI component for displaying an [antd error Alert]{@link https://ant.design/components/alert/} @@ -13,7 +24,11 @@ import { Alert } from "antd"; * @param {object} props - any other props that are passed * @returns {Element} - Returns an antd error 'Alert' component */ -export function ErrorAlert({ message, description, ...props }) { +export function ErrorAlert({ + message, + description, + ...props +}: ErrorAlertProps): JSX.Element { return ( ); -} \ No newline at end of file +} diff --git a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx index 9482b826699..28e4a7079d7 100644 --- a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx +++ b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx @@ -13,7 +13,7 @@ import React from "react"; import { useAppDispatch, useAppSelector } from "../../../hooks/useState"; import { useResetFormOnCloseModal } from "../../../hooks"; import { - SampleSequencingObject, + SampleConcatenationObject, SequencingFile, SequencingObject, useConcatenateSequencingObjectsMutation, @@ -25,6 +25,8 @@ import { } from "../sampleFilesSlice"; import { setDefaultSequencingObject } from "../sampleSlice"; import { IconArrowLeft, IconArrowRight, IconFile } from "../../icons/Icons"; +import { LoadingOutlined } from "@ant-design/icons"; +import { ErrorAlert } from "../../../components/alerts/ErrorAlert"; const { Title } = Typography; @@ -49,6 +51,11 @@ export function SampleFileConcatenate({ (state) => state.sampleFilesReducer ); const [concatenateSeqObjectFiles] = useConcatenateSequencingObjectsMutation(); + const [concatenateButtonDisabled, setConcatenateButtonDisabled] = + React.useState(true); + const [concatenating, setConcatenating] = React.useState(false); + const [concatenationError, setConcatenationError] = + React.useState(""); useResetFormOnCloseModal({ form, @@ -56,11 +63,13 @@ export function SampleFileConcatenate({ }); const concatenateFiles = () => { + setConcatenationError(""); const sequencingObjectIds = concatenateSelected.map( (seqObject: SequencingObject) => seqObject.identifier ); form.validateFields().then((values) => { + setConcatenating(true); concatenateSeqObjectFiles({ sampleId: sample.identifier, sequencingObjectIds: sequencingObjectIds, @@ -68,7 +77,7 @@ export function SampleFileConcatenate({ removeOriginals: values.remove_original_files, }) .unwrap() - .then((data: SampleSequencingObject[]) => { + .then((data: SampleConcatenationObject[]) => { let message = i18n("SampleFilesConcatenate.concatenationSuccess"); if (values.remove_original_files) { @@ -99,15 +108,18 @@ export function SampleFileConcatenate({ } dispatch(resetConcatenateSelected()); - dispatch(addToSequenceFiles({ sequenceFiles: data })); + dispatch( + addToSequenceFiles({ + sequenceFiles: data.sampleSequencingObjectFileModels, + }) + ); notification.success({ message }); form.resetFields(); setVisible(false); }) .catch((error) => { - notification.error({ - message: error, - }); + setConcatenating(false); + setConcatenationError(error.data.concatenationError); }); }); }; @@ -161,18 +173,27 @@ export function SampleFileConcatenate({ className="t-concatenate-confirm-modal" onCancel={() => { dispatch(resetConcatenateSelected()); + setConcatenationError(""); + setConcatenateButtonDisabled(true); setVisible(false); }} visible={visible} onOk={concatenateFiles} okText={i18n("SampleFilesConcatenate.okText")} cancelText={i18n("SampleFilesConcatenate.cancelText")} - okButtonProps={{ className: "t-concatenate-confirm" }} + okButtonProps={{ + className: "t-concatenate-confirm", + disabled: concatenateButtonDisabled, + icon: concatenating && , + }} cancelButtonProps={{ className: "t-concatenate-cancel" }} > {i18n("SampleFilesConcatenate.title")} + {concatenationError !== "" && ( + + )} -
    + { + const nameValue = form.getFieldValue("new_file_name"); + console.log(nameValue); + if (nameValue.length >= 3) { + setConcatenateButtonDisabled(false); + } else { + setConcatenateButtonDisabled(true); + } + }} + > ), }, + { + min: 3, + message: ( +
    + {i18n( + "SampleFilesConcatenate.nameLengthValidationError" + )} +
    + ), + }, ]} > diff --git a/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx b/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx index e8ef0dd16cf..c3646d914a9 100644 --- a/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx +++ b/src/main/webapp/resources/js/pages/analysis/components/AnalysisBioHansel.jsx @@ -32,7 +32,6 @@ export default function AnalysisBioHansel() { useEffect(() => { getBioHanselResults(); - setLoading(false); }, [analysisOutputsContext.outputs]); function getBioHanselResults() { @@ -50,9 +49,11 @@ export default function AnalysisBioHansel() { }).then(({ text }) => { const parsedResults = JSON.parse(text); setBioHanselResults(parsedResults[0]); + setLoading(false); }); } else { setBioHanselResults(undefined); + setLoading(false); } } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java index b4869fbe6ec..11c97455225 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/samples/SamplesAjaxControllerTest.java @@ -9,6 +9,7 @@ import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxResponse; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleAnalyses; +import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleConcatenationModel; import ca.corefacility.bioinformatics.irida.ria.web.samples.dto.SampleSequencingObjectFileModel; import org.apache.commons.io.IOUtils; @@ -120,7 +121,7 @@ public void testUploadSequenceFilePairsAndSingle() { @Test public void testConcatenateFiles() { - ResponseEntity> responseEntity = controller.concatenateSequenceFiles(SAMPLE.getId(), sequencingObjectIds, "newFile", false ); + ResponseEntity responseEntity = controller.concatenateSequenceFiles(SAMPLE.getId(), sequencingObjectIds, "newFile", false, Locale.ENGLISH ); assertEquals(HttpStatus.OK, responseEntity.getStatusCode(), "Response is ok"); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UISampleServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UISampleServiceTest.java index 1359773325b..70532c95f00 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UISampleServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UISampleServiceTest.java @@ -353,17 +353,17 @@ public void testConcatenateFiles() throws ConcatenateException { new SampleSequencingObjectJoin(SAMPLE_1, expectedConcatenatedFile.getObject())); when(sampleSequencingObjectJoin.getObject()).thenReturn(expectedConcatenatedFile.getObject()); - List sampleSequencingObjectFileModels = service.concatenateSequenceFiles( - SAMPLE_1.getId(), sequencingObjectsIdList, "test_file_AB", false); + SampleConcatenationModel sampleConcatenationModel = service.concatenateSequenceFiles( + SAMPLE_1.getId(), sequencingObjectsIdList, "test_file_AB", false, Locale.ENGLISH); - assertEquals(1, sampleSequencingObjectFileModels.size(), "Should have concatenated 2 single end files into 1"); + assertEquals(1, sampleConcatenationModel.getSampleSequencingObjectFileModels().size(), "Should have concatenated 2 single end files into 1"); assertEquals(expectedConcatenatedFile.getObject() .getFiles() .stream() .findFirst() .get() - .getLabel(), sampleSequencingObjectFileModels.get(0) + .getLabel(), sampleConcatenationModel.getSampleSequencingObjectFileModels().get(0) .getFileInfo() .getLabel(), "The concatenated file name should be the same as the SampleSequencingObject -> SequencingObject -> File name"); From 4473f13bb3cbf70e766c714e5e69de097eff13ca Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 11 Oct 2022 14:10:43 -0500 Subject: [PATCH 284/655] Removed logging statement --- .../js/components/samples/components/SampleFileContenate.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx index 28e4a7079d7..6307aa321e6 100644 --- a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx +++ b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx @@ -250,7 +250,6 @@ export function SampleFileConcatenate({ form={form} onFieldsChange={() => { const nameValue = form.getFieldValue("new_file_name"); - console.log(nameValue); if (nameValue.length >= 3) { setConcatenateButtonDisabled(false); } else { From 758c84bf1fea7e0d56ffab7b397e8ebdfe18d961 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 11 Oct 2022 16:26:59 -0500 Subject: [PATCH 285/655] Updated to destructure data --- .../js/components/samples/components/SampleFileContenate.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx index 6307aa321e6..feeb6621ccf 100644 --- a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx +++ b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx @@ -14,6 +14,7 @@ import { useAppDispatch, useAppSelector } from "../../../hooks/useState"; import { useResetFormOnCloseModal } from "../../../hooks"; import { SampleConcatenationObject, + SampleSequencingObject, SequencingFile, SequencingObject, useConcatenateSequencingObjectsMutation, @@ -77,7 +78,7 @@ export function SampleFileConcatenate({ removeOriginals: values.remove_original_files, }) .unwrap() - .then((data: SampleConcatenationObject[]) => { + .then(({ sampleSequencingObjectFileModels }) => { let message = i18n("SampleFilesConcatenate.concatenationSuccess"); if (values.remove_original_files) { @@ -110,7 +111,7 @@ export function SampleFileConcatenate({ dispatch(resetConcatenateSelected()); dispatch( addToSequenceFiles({ - sequenceFiles: data.sampleSequencingObjectFileModels, + sequenceFiles: sampleSequencingObjectFileModels, }) ); notification.success({ message }); From 194cb1964da936aa5d8bcb7fa888540d4d67b96c Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 11 Oct 2022 17:08:21 -0500 Subject: [PATCH 286/655] Updated file upload progress display to stay visible till the file has been uploaded and a response received --- .../samples/components/SampleFileContenate.tsx | 1 + .../js/components/samples/components/SampleFiles.tsx | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx index feeb6621ccf..2c583332dbb 100644 --- a/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx +++ b/src/main/webapp/resources/js/components/samples/components/SampleFileContenate.tsx @@ -117,6 +117,7 @@ export function SampleFileConcatenate({ notification.success({ message }); form.resetFields(); setVisible(false); + setConcatenating(false); }) .catch((error) => { setConcatenating(false); diff --git a/src/main/webapp/resources/js/components/samples/components/SampleFiles.tsx b/src/main/webapp/resources/js/components/samples/components/SampleFiles.tsx index a7562a7ab9b..55e3c25d6a6 100644 --- a/src/main/webapp/resources/js/components/samples/components/SampleFiles.tsx +++ b/src/main/webapp/resources/js/components/samples/components/SampleFiles.tsx @@ -130,8 +130,7 @@ export default function SampleFiles() { total: number; }) => { if (progressEvent.loaded === progressEvent.total) { - setSequenceFiles([]); - setSeqFileProgress(0); + setSeqFileProgress(99); } else { setSeqFileProgress( Math.round((progressEvent.loaded / progressEvent.total) * 100.0) @@ -158,6 +157,7 @@ export default function SampleFiles() { message: i18n("SampleFiles.successfullyUploaded", "sequence"), }); dispatch(addToSequenceFiles({ sequenceFiles: response })); + setSequenceFiles([]); }) .catch((error) => { if (error !== "canceled") { @@ -184,8 +184,7 @@ export default function SampleFiles() { total: number; }) => { if (progressEvent.loaded === progressEvent.total) { - setAssemblyFiles([]); - setAssemblyProgress(0); + setAssemblyProgress(99); } else { setAssemblyProgress( Math.round((progressEvent.loaded / progressEvent.total) * 100.0) @@ -212,6 +211,7 @@ export default function SampleFiles() { message: i18n("SampleFiles.successfullyUploaded", "assembly"), }); dispatch(addToAssemblyFiles({ assemblies: response })); + setAssemblyFiles([]); }) .catch((error) => { if (error !== "canceled") { @@ -238,8 +238,7 @@ export default function SampleFiles() { total: number; }) => { if (progressEvent.loaded === progressEvent.total) { - setFast5Files([]); - setFast5Progress(0); + setFast5Progress(99); } else { setFast5Progress( Math.round((progressEvent.loaded / progressEvent.total) * 100.0) @@ -266,6 +265,7 @@ export default function SampleFiles() { message: i18n("SampleFiles.successfullyUploaded", "fast5"), }); dispatch(addToFast5Files({ fast5: response })); + setFast5Files([]); }) .catch((error) => { if (error !== "canceled") { From ed048d9ce2da605477b494050b76bf62be1036d5 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 12 Oct 2022 12:34:35 -0500 Subject: [PATCH 287/655] starting to add restriction drop-down to map headers step --- .../SampleMetadataImportMapHeaders.tsx | 67 +++++++++++++------ .../components/SampleMetadataImportReview.tsx | 6 +- .../services/importReducer.ts | 22 +++++- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index ed7e68d1235..8725b9eb97a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -1,13 +1,15 @@ import React from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { Button, Radio, RadioChangeEvent, Typography } from "antd"; +import { Button, Select, Table, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; -import { BlockRadioInput } from "../../../../components/ant.design/forms/BlockRadioInput"; import { IconArrowLeft, IconArrowRight, } from "../../../../components/icons/Icons"; -import { setSampleNameColumn } from "../services/importReducer"; +import { + MetadataHeaderItem, + setSampleNameColumn, +} from "../services/importReducer"; import { NavigateFunction } from "react-router/dist/lib/hooks"; import { ImportDispatch, @@ -36,7 +38,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { React.useEffect(() => { if (!column) { - setColumn(sampleNameColumn ? sampleNameColumn : headers[0]); + setColumn(sampleNameColumn ? sampleNameColumn : headers[0]?.name); } }, [sampleNameColumn, headers, column]); @@ -48,25 +50,50 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { } }; + const columns = [ + { + title: "Headers", + dataIndex: "name", + }, + { + title: "Restriction", + dataIndex: "level", + render(id: number, item: MetadataHeaderItem) { + return ( + + ); + }, + }, + ]; + + const rowSelection = { + onChange: ( + selectedRowKeys: React.Key[], + selectedRows: MetadataHeaderItem[] + ) => { + setColumn(selectedRows[0].name); + }, + }; + return ( {i18n("SampleMetadataImportMapHeaders.description")} - setColumn(e.target.value)} - > - {headers.map((header: string, index: number) => ( - - - {header} - - - ))} - +
    row.rowKey} + rowSelection={{ + type: "radio", + ...rowSelection, + }} + columns={columns} + dataSource={headers} + pagination={false} + />
    row.rowKey} - rowSelection={{ - type: "radio", - ...rowSelection, - }} columns={columns} - dataSource={headers} + dataSource={headers.filter( + (header) => header.name !== updatedSampleNameColumn + )} pagination={false} />
    From 2f71c367f564f744c34ffdd684aaf55334204659 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 27 Oct 2022 10:34:22 -0500 Subject: [PATCH 313/655] Fixed assert equals so the expected value comes first --- .../cloud/IridaFileStorageAzureUtilityTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java index 2495579373a..3deb9df8318 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java @@ -185,7 +185,7 @@ public void testWriteFile() throws IOException { @Override public void testGetFileName() { String fileName = iridaFileStorageUtility.getFileName(PATH_TO_FASTA_FILE); - assertEquals(fileName, FILENAME, "The file names should be equal"); + assertEquals(FILENAME, fileName, "The file names should be equal"); } @Test @@ -216,7 +216,7 @@ public void testAppendToFile() throws IOException { sequenceFile.setFile(Paths.get("/opt/irida/data/test_file_1.fastq")); SequenceFile sequenceFile2 = new SequenceFile(); sequenceFile2.setFile(Paths.get("/opt/irida/data/test_file_2.fastq")); - + Long expectedFileSize = 4204L; Path temp = null; try { @@ -225,7 +225,7 @@ public void testAppendToFile() throws IOException { iridaFileStorageUtility.appendToFile(temp, sequenceFile2); iridaFileStorageUtility.writeFile(temp, PATH_TO_APPENDED_FASTQ_FILE, null, null); Long fileSizeAppendedFile = iridaFileStorageUtility.getFileSizeBytes(PATH_TO_APPENDED_FASTQ_FILE); - assertEquals(4204L, fileSizeAppendedFile, "File sizes should be equal"); + assertEquals(expectedFileSize, fileSizeAppendedFile, "File sizes should be equal"); } catch (IOException e) { throw new StorageException("Cannot file file"); } finally { @@ -246,7 +246,7 @@ public void testGetFileExtension() throws IOException { .map(SampleSequencingObjectJoin::getObject) .collect(Collectors.toList()); String fileExtension = iridaFileStorageUtility.getFileExtension(sequencingObjects); - assertEquals(fileExtension, "fastq", "Both sequencing objects should have the same file extension"); + assertEquals("fastq", fileExtension, "Both sequencing objects should have the same file extension"); } @Test @@ -256,7 +256,7 @@ public void testReadAllBytes() throws IOException { String base64EncodedLocalImage = Base64.getEncoder().encodeToString(localImageFile); byte[] azureImageFile = iridaFileStorageUtility.readAllBytes(PATH_TO_IMAGE_FILE); String base64EncodedAzureImage = Base64.getEncoder().encodeToString(azureImageFile); - assertEquals(base64EncodedAzureImage, base64EncodedLocalImage, "Bytes should be equal"); + assertEquals(base64EncodedLocalImage, base64EncodedAzureImage, "Bytes should be equal"); } @Test @@ -264,7 +264,7 @@ public void testReadAllBytes() throws IOException { public void testGetFileSizeBytes() { Long expectedFileSize = 405049L; Long fileSize = iridaFileStorageUtility.getFileSizeBytes(PATH_TO_FASTA_FILE); - assertEquals(fileSize, expectedFileSize, "File size should be equal"); + assertEquals(expectedFileSize, fileSize, "File size should be equal"); } @Test @@ -280,7 +280,7 @@ public void testReadChunk() { fileChunkResponse = iridaFileStorageUtility.readChunk(PATH_TO_FASTA_FILE, fileChunkResponse.getFilePointer(), chunk2); - assertEquals(fileChunkResponse.getText(), "CA", "Should have read the correct chunk from the file"); + assertEquals("CA", fileChunkResponse.getText(), "Should have read the correct chunk from the file"); } @Test @@ -294,7 +294,7 @@ public void testCheckWriteAccess() { @Override public void testGetStorageType() { String storageType = iridaFileStorageUtility.getStorageType(); - assertEquals(StorageType.fromString(storageType), StorageType.AZURE, "Storage type should be azure"); + assertEquals(StorageType.AZURE, StorageType.fromString(storageType), "Storage type should be azure"); } } From 58f3f9749935b2e62f7df1953226aa3664d4279f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 27 Oct 2022 10:42:46 -0500 Subject: [PATCH 314/655] Fixed assert equals so the expected value comes first --- .../cloud/IridaFileStorageAwsUtilityTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java index 44ab7055ddd..671102c9d12 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java @@ -167,7 +167,7 @@ public void testWriteFile() throws IOException { @Override public void testGetFileName() { String fileName = iridaFileStorageUtility.getFileName(PATH_TO_FASTA_FILE); - assertEquals(fileName, FILENAME, "The file names should be equal"); + assertEquals(FILENAME, fileName, "The file names should be equal"); } @Test @@ -198,6 +198,7 @@ public void testAppendToFile() throws IOException { sequenceFile.setFile(Paths.get("/opt/irida/data/test_file_1.fastq")); SequenceFile sequenceFile2 = new SequenceFile(); sequenceFile2.setFile(Paths.get("/opt/irida/data/test_file_2.fastq")); + Long expectedFileSize = 4204L; Path temp = null; @@ -207,7 +208,7 @@ public void testAppendToFile() throws IOException { iridaFileStorageUtility.appendToFile(temp, sequenceFile2); iridaFileStorageUtility.writeFile(temp, PATH_TO_APPENDED_FASTQ_FILE, null, null); Long fileSizeAppendedFile = iridaFileStorageUtility.getFileSizeBytes(PATH_TO_APPENDED_FASTQ_FILE); - assertEquals(4204L, fileSizeAppendedFile, "File sizes should be equal"); + assertEquals(expectedFileSize, fileSizeAppendedFile, "File sizes should be equal"); } catch (IOException e) { throw new StorageException("Cannot file file"); } finally { @@ -228,7 +229,7 @@ public void testGetFileExtension() throws IOException { .map(SampleSequencingObjectJoin::getObject) .collect(Collectors.toList()); String fileExtension = iridaFileStorageUtility.getFileExtension(sequencingObjects); - assertEquals(fileExtension, "fastq", "Both sequencing objects should have the same file extension"); + assertEquals("fastq", fileExtension, "Both sequencing objects should have the same file extension"); } @Test @@ -238,7 +239,7 @@ public void testReadAllBytes() throws IOException { String base64EncodedLocalImage = Base64.getEncoder().encodeToString(localImageFile); byte[] awsImageFile = iridaFileStorageUtility.readAllBytes(PATH_TO_IMAGE_FILE); String base64EncodedAwsImage = Base64.getEncoder().encodeToString(awsImageFile); - assertEquals(base64EncodedAwsImage, base64EncodedLocalImage, "Bytes should be equal"); + assertEquals(base64EncodedLocalImage, base64EncodedAwsImage, "Bytes should be equal"); } @Test @@ -246,7 +247,7 @@ public void testReadAllBytes() throws IOException { public void testGetFileSizeBytes() { Long expectedFileSize = 405049L; Long fileSize = iridaFileStorageUtility.getFileSizeBytes(PATH_TO_FASTA_FILE); - assertEquals(fileSize, expectedFileSize, "File size should be equal"); + assertEquals(expectedFileSize, fileSize, "File size should be equal"); } @Test @@ -259,11 +260,11 @@ public void testReadChunk() { Long chunk2 = 2L; FileChunkResponse fileChunkResponse = iridaFileStorageUtility.readChunk(PATH_TO_FASTA_FILE, seek, chunk1); - assertEquals(fileChunkResponse.getText(), expectedText1, "Should have read the correct chunk from the file"); + assertEquals(expectedText1, fileChunkResponse.getText(), "Should have read the correct chunk from the file"); fileChunkResponse = iridaFileStorageUtility.readChunk(PATH_TO_FASTA_FILE, fileChunkResponse.getFilePointer(), chunk2); - assertEquals(fileChunkResponse.getText(), expectedText2, "Should have read the correct chunk from the file"); + assertEquals(expectedText2, fileChunkResponse.getText(), "Should have read the correct chunk from the file"); } @Test @@ -277,6 +278,6 @@ public void testCheckWriteAccess() { @Override public void testGetStorageType() { String storageType = iridaFileStorageUtility.getStorageType(); - assertEquals(StorageType.fromString(storageType), StorageType.AWS, "Storage type should be aws"); + assertEquals(StorageType.AWS, StorageType.fromString(storageType), "Storage type should be aws"); } } From 9d35f3dc14e969e8b4f73c540a212cad3489b353 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 27 Oct 2022 10:46:33 -0500 Subject: [PATCH 315/655] Fixed missed assertequals so the expected value comes first --- .../ria/unit/cloud/IridaFileStorageAzureUtilityTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java index 3deb9df8318..b14061d3edb 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java @@ -270,17 +270,18 @@ public void testGetFileSizeBytes() { @Test @Override public void testReadChunk() { - String expectedText = "CCCGCTCGCCACGCTTTGGC"; + String expectedText1 = "CCCGCTCGCCACGCTTTGGC"; + String expectedText2 = "CA"; Long seek = 47L; Long chunk1 = 20L; Long chunk2 = 2L; FileChunkResponse fileChunkResponse = iridaFileStorageUtility.readChunk(PATH_TO_FASTA_FILE, seek, chunk1); - assertEquals(fileChunkResponse.getText(), expectedText, "Should have read the correct chunk from the file"); + assertEquals(expectedText1, fileChunkResponse.getText(), "Should have read the correct chunk from the file"); fileChunkResponse = iridaFileStorageUtility.readChunk(PATH_TO_FASTA_FILE, fileChunkResponse.getFilePointer(), chunk2); - assertEquals("CA", fileChunkResponse.getText(), "Should have read the correct chunk from the file"); + assertEquals(expectedText2, fileChunkResponse.getText(), "Should have read the correct chunk from the file"); } @Test From 7241109ff89628405a063f8d0b3f66a5e911a66f Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 27 Oct 2022 11:07:22 -0500 Subject: [PATCH 316/655] Updated to use a static const for endpoint url and bucket region --- .../irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java index 671102c9d12..e72cb17eb23 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAwsUtilityTest.java @@ -61,6 +61,9 @@ public class IridaFileStorageAwsUtilityTest implements IridaFileStorageTestUtili private static String AWS_PATH_TO_FASTA_FILE = "opt/irida/data/" + FILENAME; private static IridaFileStorageUtility iridaFileStorageUtility; + private static String ENDPOINT_URL = "http://127.0.0.1:9090/"; + private static String BUCKET_REGION = "us-east-1"; + @BeforeAll public static void setUp() { s3Client = AmazonS3ClientBuilder.standard() @@ -68,7 +71,7 @@ public static void setUp() { .withCredentials(new AWSStaticCredentialsProvider( new AnonymousAWSCredentials())) // use any credentials here for mocking .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration("http://127.0.0.1:9090/", "us-east-1")) + new AwsClientBuilder.EndpointConfiguration(ENDPOINT_URL, BUCKET_REGION)) .build(); iridaFileStorageUtility = new IridaFileStorageAwsUtilityImpl(s3Client, bucketName); From 53a06ca04e949b50a7d0b4aeb92a659ca039ec67 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Thu, 27 Oct 2022 11:13:40 -0500 Subject: [PATCH 317/655] Updated test class so all blobs and container are deleted on teardown --- .../ria/unit/cloud/IridaFileStorageAzureUtilityTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java index b14061d3edb..b15659b9d06 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/cloud/IridaFileStorageAzureUtilityTest.java @@ -46,6 +46,7 @@ public class IridaFileStorageAzureUtilityTest implements IridaFileStorageTestUti private String DIRECTORY_PREFIX = "text-prefix-"; private static Path PATH_TO_APPENDED_FASTQ_FILE = Paths.get("/opt/irida/data/iridatestfileappend.fastq"); + private static Path AZURE_PATH_TO_APPENDED_FASTQ_FILE = Paths.get("opt/irida/data/iridatestfileappend.fastq"); private static String LOCAL_RESOURCES_FASTA_FILE_PATH = "src/test/resources/files/" + FILENAME; private static String LOCAL_RESOURCES_IMAGE_FILE_PATH = "src/test/resources/files/perBaseQualityScoreChart.png"; @@ -111,8 +112,10 @@ public static void tearDown() { blobClient = containerClient.getBlobClient(AZURE_PATH_FASTQ_2); blobClient.deleteIfExists(); - blobClient = containerClient.getBlobClient(PATH_TO_APPENDED_FASTQ_FILE.toString()); + blobClient = containerClient.getBlobClient(AZURE_PATH_TO_APPENDED_FASTQ_FILE.toString()); blobClient.deleteIfExists(); + + containerClient.deleteIfExists(); } @Test @@ -239,7 +242,7 @@ public void testAppendToFile() throws IOException { @Override public void testGetFileExtension() throws IOException { Sample sample = new Sample("Sample1"); - List fileNames = List.of("test_file_1.fastq", "test_file_2.fastq"); + List fileNames = List.of("/opt/irida/data/test_file_1.fastq", "/opt/irida/data/test_file_2.fastq"); List sequencingObject = TestDataFactory.generateSingleFileSequencingObjectsForSample( sample, fileNames); List sequencingObjects = sequencingObject.stream() From 8da49bc20b959f9b392d87418f399da8c1c952f9 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 27 Oct 2022 11:52:32 -0500 Subject: [PATCH 318/655] changing updatedHeaders to useRef from useState --- .../components/SampleMetadataImportMapHeaders.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index a4844b4e5d2..29e7a6a8c47 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -38,7 +38,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { ); const [updatedSampleNameColumn, setUpdatedSampleNameColumn] = React.useState(); - const updatedHeaders: MetadataHeaderItem[] = [...headers]; + const updatedHeaders = React.useRef(headers); const dispatch: ImportDispatch = useImportDispatch(); React.useEffect(() => { @@ -66,7 +66,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { await dispatch( setSampleNameColumn({ projectId, updatedSampleNameColumn }) ); - await dispatch(updateHeaders(updatedHeaders)); + await dispatch(updateHeaders(updatedHeaders.current)); navigate(`/${projectId}/sample-metadata/upload/review`); } }; @@ -76,13 +76,13 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { }; const onRestrictionChange = (item: MetadataHeaderItem, value: string) => { - const index = updatedHeaders.findIndex( + const index = updatedHeaders.current.findIndex( (header) => header.rowKey === item.rowKey ); if (index !== -1) { - const updatedHeadersItem = { ...updatedHeaders[index] }; + const updatedHeadersItem = { ...updatedHeaders.current[index] }; updatedHeadersItem.restriction = value; - updatedHeaders[index] = updatedHeadersItem; + updatedHeaders.current[index] = updatedHeadersItem; } }; From cefa13b7d48fb48e09901d2629832b413e65f544 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 27 Oct 2022 11:58:31 -0500 Subject: [PATCH 319/655] removing preselecting a sample name column and showing table empty on load --- .../SampleMetadataImportMapHeaders.tsx | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 29e7a6a8c47..198ed60ac5f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -47,19 +47,6 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { }); }, []); - React.useEffect(() => { - if (!updatedSampleNameColumn && headers.length > 0) { - if (sampleNameColumn) { - const column = headers.filter( - (header) => header.name === sampleNameColumn - ); - setUpdatedSampleNameColumn(column[0].name); - } else { - setUpdatedSampleNameColumn(headers[0].name); - } - } - }, [updatedSampleNameColumn, headers, sampleNameColumn]); - const onSubmit = async () => { if (projectId && updatedSampleNameColumn) { setLoading(true); @@ -128,9 +115,13 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { className="t-metadata-uploader-header-table" rowKey={(row) => row.rowKey} columns={columns} - dataSource={headers.filter( - (header) => header.name !== updatedSampleNameColumn - )} + dataSource={ + updatedSampleNameColumn + ? headers.filter( + (header) => header.name !== updatedSampleNameColumn + ) + : undefined + } pagination={false} />
    From 08a7fd58daeadea5c6cf010c09ba782e9408213d Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 27 Oct 2022 12:07:36 -0500 Subject: [PATCH 320/655] diplaying table on sample name column instead of showing table empty --- .../SampleMetadataImportMapHeaders.tsx | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 198ed60ac5f..a81cee7cb1a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -33,7 +33,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { const navigate: NavigateFunction = useNavigate(); const [loading, setLoading] = React.useState(false); const [restrictions, setRestrictions] = React.useState([]); - const { headers, sampleNameColumn } = useImportSelector( + const { headers } = useImportSelector( (state: ImportState) => state.importReducer ); const [updatedSampleNameColumn, setUpdatedSampleNameColumn] = @@ -111,19 +111,17 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { ))} -
    row.rowKey} - columns={columns} - dataSource={ - updatedSampleNameColumn - ? headers.filter( - (header) => header.name !== updatedSampleNameColumn - ) - : undefined - } - pagination={false} - /> + {updatedSampleNameColumn && ( +
    row.rowKey} + columns={columns} + dataSource={headers.filter( + (header) => header.name !== updatedSampleNameColumn + )} + pagination={false} + /> + )}
    } diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 698ec7f71d4..7d68f77c864 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -10,14 +10,14 @@ import { MetadataHeaderItem, setSampleNameColumn, updateHeaders, -} from "../services/importReducer"; +} from "../redux/importReducer"; import { NavigateFunction } from "react-router/dist/lib/hooks"; import { ImportDispatch, ImportState, useImportDispatch, useImportSelector, -} from "../store"; +} from "../redux/store"; import { getMetadataRestrictions } from "../../../../apis/metadata/field"; const { Text } = Typography; @@ -38,7 +38,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { ); const [updatedSampleNameColumn, setUpdatedSampleNameColumn] = React.useState(sampleNameColumn); - const updatedHeaders = React.useRef(headers); + const updatedHeaders: MetadataHeaderItem[] = [...headers]; const dispatch: ImportDispatch = useImportDispatch(); React.useEffect(() => { @@ -53,7 +53,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { await dispatch( setSampleNameColumn({ projectId, updatedSampleNameColumn }) ); - await dispatch(updateHeaders(updatedHeaders.current)); + await dispatch(updateHeaders(updatedHeaders)); navigate(`/${projectId}/sample-metadata/upload/review`); } }; @@ -63,13 +63,13 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { }; const onRestrictionChange = (item: MetadataHeaderItem, value: string) => { - const index = updatedHeaders.current.findIndex( + const index = updatedHeaders.findIndex( (header) => header.rowKey === item.rowKey ); if (index !== -1) { - const updatedHeadersItem = { ...updatedHeaders.current[index] }; + const updatedHeadersItem = { ...updatedHeaders[index] }; updatedHeadersItem.restriction = value; - updatedHeaders.current[index] = updatedHeadersItem; + updatedHeaders[index] = updatedHeadersItem; } }; @@ -116,8 +116,8 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { className="t-metadata-uploader-header-table" rowKey={(row) => row.rowKey} columns={columns} - dataSource={headers.filter( - (header) => header.name !== updatedSampleNameColumn + dataSource={updatedHeaders.filter( + (updatedHeader) => updatedHeader.name !== updatedSampleNameColumn )} pagination={false} scroll={{ y: 600 }} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx index 61de3c69a74..ba7ce206aac 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx @@ -16,7 +16,7 @@ import { IconExclamationCircle, } from "../../../../components/icons/Icons"; import styled from "styled-components"; -import { saveMetadata } from "../services/importReducer"; +import { saveMetadata } from "../redux/importReducer"; import { getPaginationOptions } from "../../../../utilities/antdesign-table-utilities"; import { NavigateFunction } from "react-router/dist/lib/hooks"; import { @@ -24,7 +24,7 @@ import { ImportState, useImportDispatch, useImportSelector, -} from "../store"; +} from "../redux/store"; import { MetadataItem } from "../../../../apis/projects/samples"; import { ColumnsType, ColumnType } from "antd/es/table"; import { TableRowSelection } from "antd/lib/table/interface"; diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx index 48bd36f2044..6f0995ca1bb 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx @@ -1,16 +1,12 @@ import React from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { - setHeaders, - setMetadata, - setProjectId, -} from "../services/importReducer"; +import { setHeaders, setMetadata, setProjectId } from "../redux/importReducer"; import { notification, Spin, StepsProps, Typography } from "antd"; import { DragUpload } from "../../../../components/files/DragUpload"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import * as XLSX from "xlsx"; import { WorkBook } from "xlsx"; -import { ImportDispatch, useImportDispatch } from "../store"; +import { ImportDispatch, useImportDispatch } from "../redux/store"; import { NavigateFunction } from "react-router/dist/lib/hooks"; import { MetadataItem } from "../../../../apis/projects/samples"; import { RcFile, UploadFileStatus } from "antd/lib/upload/interface"; diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js index bdae61a979b..a2a170c0ecc 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js @@ -7,7 +7,7 @@ import { SampleMetadataImportMapHeaders } from "./components/SampleMetadataImpor import { SampleMetadataImportReview } from "./components/SampleMetadataImportReview"; import { SampleMetadataImportUploadFile } from "./components/SampleMetadataImportUploadFile"; import { setBaseUrl } from "../../../utilities/url-utilities"; -import store from "./store"; +import store from "./redux/store"; /* Router for sample metadata importer. diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts similarity index 94% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts index 760b0b792f7..6ad278af408 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts @@ -13,7 +13,7 @@ import { validateSamples, ValidateSamplesResponse, } from "../../../../apis/projects/samples"; -import { ImportDispatch, ImportState } from "../store"; +import { ImportDispatch, ImportState } from "./store"; import { getMetadataFieldsForProject, MetadataField, @@ -280,6 +280,14 @@ export const setMetadataSaveDetails = createAction( }) ); +/* +Redux action for resetting the state. +For more information on redux actions see: https://redux-toolkit.js.org/api/createAction + */ +export const resetImport = createAction(`importReducer/resetImport`, () => ({ + payload: {}, +})); + /* Redux reducer for project metadata. For more information on redux reducers see: https://redux-toolkit.js.org/api/createReducer @@ -307,4 +315,12 @@ export const importReducer = createReducer(initialState, (builder) => { builder.addCase(saveMetadata.fulfilled, (state, action) => { state.metadataSaveDetails = action.payload.metadataSaveDetails; }); + builder.addCase(resetImport, (state, action) => { + state.projectId = ""; + state.sampleNameColumn = ""; + state.headers = []; + state.metadata = []; + state.metadataValidateDetails = {}; + state.metadataSaveDetails = {}; + }); }); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts similarity index 91% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts index 87c75c319f6..cd7a6401e75 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/store.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts @@ -4,9 +4,9 @@ import { Dispatch, MiddlewareAPI, } from "@reduxjs/toolkit"; -import { importReducer } from "./services/importReducer"; +import { importReducer } from "./importReducer"; import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux"; -import { getProjectIdFromUrl } from "../../../utilities/url-utilities"; +import { getProjectIdFromUrl } from "../../../../utilities/url-utilities"; const storageKey = "metadataImport"; From 7053209910584ac3aae64f70f5ef393960673bf4 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 31 Oct 2022 14:30:46 -0500 Subject: [PATCH 340/655] Updated to start and kill docker container for filesystem tests --- run-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run-tests.sh b/run-tests.sh index b2278bec869..7e255ca2d6e 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -11,6 +11,7 @@ JDBC_URL=jdbc:mysql://$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME TMP_DIRECTORY=`mktemp -d /tmp/irida-test-XXXXXXXX` chmod 777 $TMP_DIRECTORY # Needs to be world-accessible so that Docker/Galaxy can access +AZURITE_DOCKER_NAME=irida-selenium-azurite GALAXY_DOCKER=phacnml/galaxy-irida-20.09:21.05.2-it GALAXY_DOCKER_NAME=irida-galaxy-test GALAXY_PORT=48889 @@ -105,8 +106,10 @@ test_service() { } test_file_system() { + docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 --name $AZURITE_DOCKER_NAME mcr.microsoft.com/azure-storage/azurite ./gradlew clean check fileSystemITest -Dspring.datasource.url=$JDBC_URL -Dfile.processing.decompress=true -Dirida.it.rootdirectory=$TMP_DIRECTORY -Dspring.datasource.dbcp2.max-wait=$DB_MAX_WAIT_MILLIS $@ exit_code=$? + docker rm -f -v $AZURITE_DOCKER_NAME; return $exit_code } From 2fc03df33bc27cf6eb4bd2e7a2f1b2965ed448f7 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 31 Oct 2022 14:45:57 -0500 Subject: [PATCH 341/655] Removed commented out service --- .github/workflows/ci-test.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b86a0c48c41..892c159e647 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -40,14 +40,7 @@ jobs: "open_api_testing", "file_system_testing" ] -# services: -# azurite: -# if: matrix.suite == 'file_system_testing' && always() -# image: mcr.microsoft.com/azure-storage/azurite -# ports: -# - 10000:10000 -# - 10001:10001 -# - 10002:10002 + steps: - uses: actions/checkout@v3 #Checkout the project from git - name: Get pnpm store directory From 66d7037e54ec55f35d70fa0ee2023f31d4ab9dd4 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Mon, 31 Oct 2022 14:49:36 -0500 Subject: [PATCH 342/655] Updated to start and kill docker container for filesystem tests --- run-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run-tests.sh b/run-tests.sh index fa464e173c7..1e6a516781c 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -11,6 +11,7 @@ JDBC_URL=jdbc:mysql://$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME TMP_DIRECTORY=`mktemp -d /tmp/irida-test-XXXXXXXX` chmod 777 $TMP_DIRECTORY # Needs to be world-accessible so that Docker/Galaxy can access +S3MOCK_DOCKER_NAME=irida-selenium-s3 GALAXY_DOCKER=phacnml/galaxy-irida-20.09:21.05.2-it GALAXY_DOCKER_NAME=irida-galaxy-test GALAXY_PORT=48889 @@ -105,8 +106,10 @@ test_service() { } test_file_system() { + docker run -d -p 9090:9090 -p 9191:9191 --name $S3MOCK_DOCKER_NAME -t adobe/s3mock ./gradlew clean check fileSystemITest -Dspring.datasource.url=$JDBC_URL -Dfile.processing.decompress=true -Dirida.it.rootdirectory=$TMP_DIRECTORY -Dspring.datasource.dbcp2.max-wait=$DB_MAX_WAIT_MILLIS $@ exit_code=$? + docker rm -f -v $S3MOCK_DOCKER_NAME return $exit_code } From 5b83d3551eef0e07d45558e0011c9a64dc27a3bc Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Nov 2022 10:39:22 -0500 Subject: [PATCH 343/655] Removed starting up of docker container for azurite from the ci-test file as we start it and remove it within the test setup for the filesystem in the run-tests script --- .github/workflows/ci-test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 892c159e647..92757743d9e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -77,9 +77,6 @@ jobs: while ! mysqladmin ping --host=${{ env.MYSQL_HOST }} --port=${{ env.MYSQL_PORT }} --user=${{ env.MYSQL_USER }} --password=${{ env.MYSQL_ROOT_PASSWORD }} --silent; do sleep 1 done - - name: Pull and Start Azurite Docker Container - run: docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite - if: matrix.suite == 'file_system_testing' && always() - name: Run tests with Gradle # Run the test suite run: ./run-tests.sh --db-host ${{ env.MYSQL_HOST }} --db-port ${{ env.MYSQL_PORT }} --database ${{ env.MYSQL_DATABASE}} ${{ matrix.suite }} - name: Publish Test Report From 8090834e0dfa96b94979447088659adbaed800a3 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Nov 2022 12:03:09 -0500 Subject: [PATCH 344/655] Changed azurite docker container name --- run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-tests.sh b/run-tests.sh index 7e255ca2d6e..bed6a5b5afc 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -11,7 +11,7 @@ JDBC_URL=jdbc:mysql://$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME TMP_DIRECTORY=`mktemp -d /tmp/irida-test-XXXXXXXX` chmod 777 $TMP_DIRECTORY # Needs to be world-accessible so that Docker/Galaxy can access -AZURITE_DOCKER_NAME=irida-selenium-azurite +AZURITE_DOCKER_NAME=irida-docker-azurite GALAXY_DOCKER=phacnml/galaxy-irida-20.09:21.05.2-it GALAXY_DOCKER_NAME=irida-galaxy-test GALAXY_PORT=48889 From 562450b6a084f4ecf8abdee2c78042abdc56ebeb Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Nov 2022 12:57:49 -0500 Subject: [PATCH 345/655] Removed file no longer used --- .../webapp/pages/samples/sample_files.html | 431 ------------------ 1 file changed, 431 deletions(-) delete mode 100644 src/main/webapp/pages/samples/sample_files.html diff --git a/src/main/webapp/pages/samples/sample_files.html b/src/main/webapp/pages/samples/sample_files.html deleted file mode 100644 index ace32eed585..00000000000 --- a/src/main/webapp/pages/samples/sample_files.html +++ /dev/null @@ -1,431 +0,0 @@ - - - - THIS IS SOMETHING WRONG - - - - - -
    -
    - -
    -
    -
    - - -
    -
    -
    -
    -
    -
    - -
    -
    -

    -

    -
    -
    -
    -
    -
    -
    -

    Sequence Files

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    -
    -     		_Assembly__Status_ -
    -
    -
    -     		_SISTR__Status_ - -
    -
    -
    -     		-
      -
    • - -
      • -
    -
    -
    - - - - - - -
    -
    -
    -
    -     		-
      -
    • - -
      • -
    -
    -
    -
    - -
    -
    -

    FAST5 Files

    - - - - - - - - -
    - - -
    - -
    -     		-
    -
    -
    File Size
    -
    -     		-
    -
    -
    Created Date
    -
    -     		-
    - - - - - - -
    -
    -
    -
    - -
    -
    -
    -

    -
    -
    -
    - -
    -
    - - - - - - - - -
    -
    -
    -     		- -
    -
    -
    -     		-
    -
    -
    File Size
    -
    -     		-
    -
    -
    Created Date
    -
    -     		-
    - - - - - -
    -
    -
    -
    - -
    - - - - - - - - - - - From 04f38c6a04325ec43f5de7e1bc1941947da43995 Mon Sep 17 00:00:00 2001 From: Deep Sidhu Date: Tue, 1 Nov 2022 13:00:50 -0500 Subject: [PATCH 346/655] Changed s3 mock docker container name. Removed starting of s3Mock docker container from ci-test file as we start/remove it in the filesystem test setup in the run-tests script --- .github/workflows/ci-test.yml | 3 --- run-tests.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index abfa0008639..4212b82f161 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -77,9 +77,6 @@ jobs: while ! mysqladmin ping --host=${{ env.MYSQL_HOST }} --port=${{ env.MYSQL_PORT }} --user=${{ env.MYSQL_USER }} --password=${{ env.MYSQL_ROOT_PASSWORD }} --silent; do sleep 1 done - - name: Pull and Start s3Mock Docker Container - run: docker run -d -p 9090:9090 -p 9191:9191 -t adobe/s3mock - if: matrix.suite == 'file_system_testing' && always() - name: Run tests with Gradle # Run the test suite run: ./run-tests.sh --db-host ${{ env.MYSQL_HOST }} --db-port ${{ env.MYSQL_PORT }} --database ${{ env.MYSQL_DATABASE}} ${{ matrix.suite }} - name: Publish Test Report diff --git a/run-tests.sh b/run-tests.sh index 1e6a516781c..b51c6b937ba 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -11,7 +11,7 @@ JDBC_URL=jdbc:mysql://$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME TMP_DIRECTORY=`mktemp -d /tmp/irida-test-XXXXXXXX` chmod 777 $TMP_DIRECTORY # Needs to be world-accessible so that Docker/Galaxy can access -S3MOCK_DOCKER_NAME=irida-selenium-s3 +S3MOCK_DOCKER_NAME=irida-docker-s3Mock GALAXY_DOCKER=phacnml/galaxy-irida-20.09:21.05.2-it GALAXY_DOCKER_NAME=irida-galaxy-test GALAXY_PORT=48889 From 228e414a5aeb6e80ae99e70d66302cb481f9af77 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 1 Nov 2022 15:20:49 -0500 Subject: [PATCH 347/655] assuming metadata fields with restrictions are already created before metadata entry creation --- .../web/services/UIProjectSampleService.java | 20 ++++++++++++------- .../redux/importReducer.ts | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index 09367924130..eb599d11429 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -11,7 +11,6 @@ import org.springframework.transaction.annotation.Transactional; import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException; -import ca.corefacility.bioinformatics.irida.model.enums.ProjectMetadataRole; import ca.corefacility.bioinformatics.irida.model.joins.Join; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField; @@ -145,7 +144,7 @@ public ResponseEntity createSample(CreateSampleRequest request, Lo Join join = projectService.addSampleToProjectWithoutEvent(project, sample, true); if (request.getMetadata() != null) { Set metadataEntrySet = createMetadata(request.getMetadata(), project); - sampleService.mergeSampleMetadata(sample, metadataEntrySet); + sampleService.updateSampleMetadata(sample, metadataEntrySet); } return ResponseEntity.ok(new AjaxCreateItemSuccessResponse(join.getObject().getId())); } catch (EntityNotFoundException e) { @@ -178,7 +177,7 @@ public ResponseEntity updateSample(UpdateSampleRequest request, Lo } if (request.getMetadata() != null) { Set metadataEntrySet = createMetadata(request.getMetadata(), project); - sampleService.updateSampleMetadata(sample, metadataEntrySet); + sampleService.mergeSampleMetadata(sample, metadataEntrySet); } sampleService.update(sample); return ResponseEntity.ok(new AjaxUpdateItemSuccessResponse( @@ -197,10 +196,17 @@ public ResponseEntity updateSample(UpdateSampleRequest request, Lo */ private Set createMetadata(List metadataFields, Project project) { Set metadataEntrySet = metadataFields.stream().map(entry -> { - MetadataTemplateField field = metadataTemplateService.saveMetadataField( - new MetadataTemplateField(entry.getField(), "text")); - ProjectMetadataRole role = ProjectMetadataRole.fromString(entry.getRestriction()); - metadataTemplateService.setMetadataRestriction(project, field, role); + String label = entry.getField(); + MetadataTemplateField field = metadataTemplateService.readMetadataFieldByLabel(label); + // MetadataTemplateField field; + // MetadataTemplateField existingField = metadataTemplateService.readMetadataFieldByLabel(label); + // if (existingField != null) { + // field = existingField; + // } else { + // field = metadataTemplateService.saveMetadataField(new MetadataTemplateField(label, "text")); + // } + // ProjectMetadataRole role = ProjectMetadataRole.fromString(entry.getRestriction()); + // metadataTemplateService.setMetadataRestriction(project, field, role); return new MetadataEntry(entry.getValue(), "text", field); }).collect(Collectors.toSet()); return metadataEntrySet; diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts index 6ad278af408..5a43a22cd05 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts @@ -315,7 +315,7 @@ export const importReducer = createReducer(initialState, (builder) => { builder.addCase(saveMetadata.fulfilled, (state, action) => { state.metadataSaveDetails = action.payload.metadataSaveDetails; }); - builder.addCase(resetImport, (state, action) => { + builder.addCase(resetImport, (state) => { state.projectId = ""; state.sampleNameColumn = ""; state.headers = []; From 2f05516c52abd3a2c2adf53da4f733a26ae0430b Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 1 Nov 2022 17:12:43 -0500 Subject: [PATCH 348/655] starting to create a new api endpoint to create metadata fields with restrictions --- .../ajax/metadata/MetadataAjaxController.java | 27 +++++--- .../ria/web/services/UIMetadataService.java | 65 ++++++++++++------- .../resources/js/apis/metadata/field.ts | 20 +++++- .../redux/importReducer.ts | 13 ++++ 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java index 62b689fe299..1fe093d7d39 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java @@ -47,13 +47,13 @@ public ResponseEntity> getProjectMetadataTemplates /** * Create a new metadata template within a project * - * @param template details about the template to create + * @param template details about the template to create * @param projectId identifier for a project * @return the newly created {@link MetadataTemplate} */ @PostMapping("/templates") - public ResponseEntity createNewMetadataTemplate( - @RequestBody MetadataTemplate template, @RequestParam Long projectId) { + public ResponseEntity createNewMetadataTemplate(@RequestBody MetadataTemplate template, + @RequestParam Long projectId) { return ResponseEntity.ok(service.createMetadataTemplate(template, projectId)); } @@ -61,7 +61,7 @@ public ResponseEntity createNewMetadataTemplate( * Updated the fields in a {@link MetadataTemplate} * * @param template the updated template to save - * @param locale Current users {@link Locale} + * @param locale Current users {@link Locale} * @return Message for UI to display about the result of the update. */ @PutMapping("/templates/{templateId}") @@ -89,8 +89,7 @@ public ResponseEntity deleteMetadataTemplate(@PathVariable Long te return ResponseEntity.ok( new AjaxSuccessResponse(service.deleteMetadataTemplate(templateId, projectId, locale))); } catch (Exception e) { - return ResponseEntity.status(HttpStatus.NOT_FOUND) - .body(new AjaxErrorResponse(e.getMessage())); + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(new AjaxErrorResponse(e.getMessage())); } } @@ -105,6 +104,19 @@ public List getMetadataFieldsForProject(@RequestParam Long return service.getMetadataFieldsForProject(projectId); } + /** + * Create project metadata fields with restrictions (no metadata entries) + * + * @param projectId Identifier for a project + * @param fields List of project metadata fields + * @return list of {@link MetadataTemplateField}s + */ + @PostMapping("/fields") + public ResponseEntity createMetadataFieldsForProject(@RequestParam Long projectId, + @RequestBody List fields) { + return ResponseEntity.ok(new AjaxSuccessResponse(service.createMetadataFieldsForProject(projectId, fields))); + } + /** * Get all the metadata fields for a list of projects * @@ -132,8 +144,7 @@ public ResponseEntity setDefaultMetadataTemplate(@PathVariable Lon return ResponseEntity.ok( new AjaxSuccessResponse(service.setDefaultMetadataTemplate(templateId, projectId, locale))); } catch (Exception e) { - return ResponseEntity.status(HttpStatus.NOT_FOUND) - .body(new AjaxErrorResponse(e.getMessage())); + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(new AjaxErrorResponse(e.getMessage())); } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java index ebac150569f..6519941d373 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java @@ -11,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.stereotype.Component; @@ -57,15 +56,12 @@ public List getProjectMetadataTemplates(Long projectId) Project project = projectService.read(projectId); List templates = templateService.getMetadataTemplatesForProject(project); - return templates.stream() - .map(template -> { - List permittedFieldsForTemplate = templateService.getPermittedFieldsForTemplate( - template); - List fields = addRestrictionsToMetadataFields(project, - permittedFieldsForTemplate); - return new ProjectMetadataTemplate(template, fields); - }) - .collect(Collectors.toList()); + return templates.stream().map(template -> { + List permittedFieldsForTemplate = templateService.getPermittedFieldsForTemplate( + template); + List fields = addRestrictionsToMetadataFields(project, permittedFieldsForTemplate); + return new ProjectMetadataTemplate(template, fields); + }).collect(Collectors.toList()); } /** @@ -141,6 +137,31 @@ public List getMetadataFieldsForProject(Long projectId) { return addRestrictionsToMetadataFields(project, fields); } + /** + * Create project metadata fields with restrictions (no metadata entries) + * + * @param projectId Identifier for a {@link Project} + * @param fields List of project metadata fields + * @return result message + */ + @Transactional + public String createMetadataFieldsForProject(Long projectId, List fields) { + Project project = projectService.read(projectId); + for (ProjectMetadataField field : fields) { + String label = field.getLabel(); + MetadataTemplateField templateField; + MetadataTemplateField existingTemplateField = templateService.readMetadataFieldByLabel(label); + if (existingTemplateField != null) { + templateField = existingTemplateField; + } else { + templateField = templateService.saveMetadataField(new MetadataTemplateField(label, "text")); + } + ProjectMetadataRole role = ProjectMetadataRole.fromString(field.getRestriction()); + templateService.setMetadataRestriction(project, templateField, role); + } + return "SUCCESS"; + } + /** * Get all {@link MetadataTemplateField}s belonging to a list of {@link Project}s * @@ -154,13 +175,11 @@ public List getMetadataFieldsForProjects(List projec List fields = templateService.getPermittedFieldsForCurrentUser(project, false); projectMetadataFieldList = Stream.concat(projectMetadataFieldList.stream(), - addRestrictionsToMetadataFields(project, fields).stream()) - .collect(Collectors.toList()); + addRestrictionsToMetadataFields(project, fields).stream()).collect(Collectors.toList()); } // Sort in descending order by restriction and use distinct to get unique metadata template fields - projectMetadataFieldList.sort(Comparator.comparing(ProjectMetadataField::getRestriction) - .reversed()); + projectMetadataFieldList.sort(Comparator.comparing(ProjectMetadataField::getRestriction).reversed()); projectMetadataFieldList = projectMetadataFieldList.stream() .filter(distinctByKey(ProjectMetadataField::getLabel)) .collect(Collectors.toList()); @@ -194,7 +213,8 @@ public String updateMetadataProjectField(Long projectId, Long fieldId, ProjectMe Project project = projectService.read(projectId); MetadataTemplateField field = templateService.readMetadataField(fieldId); templateService.setMetadataRestriction(project, field, newRole); - return messageSource.getMessage("server.MetadataFieldsListManager.update", new Object[] { field.getLabel(), + return messageSource.getMessage("server.MetadataFieldsListManager.update", new Object[] { + field.getLabel(), messageSource.getMessage("metadataRole." + newRole.toString(), new Object[] {}, locale) }, locale); } @@ -233,9 +253,7 @@ public String setDefaultMetadataTemplate(Long templateId, Long projectId, Locale */ private List addRestrictionsToMetadataFields(Project project, List fields) { - return fields.stream() - .map(field -> createProjectMetadataField(project, field)) - .collect(Collectors.toList()); + return fields.stream().map(field -> createProjectMetadataField(project, field)).collect(Collectors.toList()); } /** @@ -252,7 +270,8 @@ public List getProjectMetadataRoles(Locale locale) { } /** - * Utility function to update a specific {@link MetadataTemplateField} with its security restrictions for a project. + * Utility function to update a specific {@link MetadataTemplateField} with its security restrictions for a + * project. * * @param project The {@link Project} the fields belong to * @param field the {@link MetadataTemplateField} to update @@ -261,15 +280,13 @@ public List getProjectMetadataRoles(Locale locale) { private ProjectMetadataField createProjectMetadataField(Project project, MetadataTemplateField field) { MetadataRestriction restriction = templateService.getMetadataRestrictionForFieldAndProject(project, field); //default to LEVEL_1 if no restriction is set - String level = restriction == null ? - ProjectMetadataRole.LEVEL_1.toString() : - restriction.getLevel() - .toString(); + String level = restriction == null ? ProjectMetadataRole.LEVEL_1.toString() : restriction.getLevel().toString(); return new ProjectMetadataField(field, level); } /** - * Predicate that maintains state about what it's seen previously, and that returns whether the given element was seen for the first time: + * Predicate that maintains state about what it's seen previously, and that returns whether the given element was + * seen for the first time: * * @param keyExtractor * @param diff --git a/src/main/webapp/resources/js/apis/metadata/field.ts b/src/main/webapp/resources/js/apis/metadata/field.ts index c58d5eb7fdd..909c2e439e8 100644 --- a/src/main/webapp/resources/js/apis/metadata/field.ts +++ b/src/main/webapp/resources/js/apis/metadata/field.ts @@ -9,10 +9,10 @@ import { setBaseUrl } from "../../utilities/url-utilities"; const BASE_URL = setBaseUrl(`/ajax/metadata/fields`); export interface MetadataField { - id: number; - fieldKey: string; + id?: number; + fieldKey?: string; label: string; - type: string; + type?: string; restriction: string; } @@ -126,3 +126,17 @@ export async function getAllMetadataFieldsForProjects( } } } + +/* + * Create metadata fields for a specific project + * @returns {Promise} + */ +export async function createMetadataFieldsForProject({ + projectId, + body, +}: { + projectId: string; + body: MetadataField[]; +}) { + return await axios.post(`${BASE_URL}?projectId=${projectId}`, body); +} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts index 5a43a22cd05..b7bba45efee 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts @@ -15,6 +15,7 @@ import { } from "../../../../apis/projects/samples"; import { ImportDispatch, ImportState } from "./store"; import { + createMetadataFieldsForProject, getMetadataFieldsForProject, MetadataField, } from "../../../../apis/metadata/field"; @@ -77,6 +78,18 @@ export const saveMetadata = createAsyncThunk< state.importReducer; const metadataSaveDetails: Record = {}; + const body = headers.map((header) => ({ + label: header.name, + restriction: header.restriction, + })); + console.log("body"); + console.log(body); + + await createMetadataFieldsForProject({ + projectId, + body, + }); + const chunkSize = 100; for (let i = 0; i < metadata.length; i = i + chunkSize) { const promises: Promise[] = []; From 2af350714558d8f727d4e314da43de84f1ae8bd6 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 2 Nov 2022 10:13:15 -0500 Subject: [PATCH 349/655] fixing create metadata fields api endpoint --- .../web/ajax/metadata/MetadataAjaxController.java | 6 ++++-- .../ajax/metadata/dto/ProjectMetadataField.java | 8 +++++--- .../irida/ria/web/services/UIMetadataService.java | 5 +++-- src/main/resources/i18n/messages.properties | 1 + .../samples-metadata-import/redux/importReducer.ts | 14 ++++++-------- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java index 1fe093d7d39..a1a0d0c5098 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/MetadataAjaxController.java @@ -109,12 +109,14 @@ public List getMetadataFieldsForProject(@RequestParam Long * * @param projectId Identifier for a project * @param fields List of project metadata fields + * @param locale Current users {@link Locale} * @return list of {@link MetadataTemplateField}s */ @PostMapping("/fields") public ResponseEntity createMetadataFieldsForProject(@RequestParam Long projectId, - @RequestBody List fields) { - return ResponseEntity.ok(new AjaxSuccessResponse(service.createMetadataFieldsForProject(projectId, fields))); + @RequestBody List fields, Locale locale) { + return ResponseEntity.ok( + new AjaxSuccessResponse(service.createMetadataFieldsForProject(projectId, fields, locale))); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/dto/ProjectMetadataField.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/dto/ProjectMetadataField.java index 8683fe55c0c..b756d3d897b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/dto/ProjectMetadataField.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/metadata/dto/ProjectMetadataField.java @@ -3,9 +3,8 @@ import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField; /** - * A representation of a {@link MetadataTemplateField} specifically for a project. - * This class was required since metadata fields for a project need to include the restriction - * level specific for that project. + * A representation of a {@link MetadataTemplateField} specifically for a project. This class was required since + * metadata fields for a project need to include the restriction level specific for that project. */ public class ProjectMetadataField { private Long id; @@ -14,6 +13,9 @@ public class ProjectMetadataField { private String type; private String restriction; + public ProjectMetadataField() { + } + public ProjectMetadataField(MetadataTemplateField field, String restriction) { this.id = field.getId(); this.fieldKey = field.getFieldKey(); diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java index 6519941d373..dcb4e885a82 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIMetadataService.java @@ -142,10 +142,11 @@ public List getMetadataFieldsForProject(Long projectId) { * * @param projectId Identifier for a {@link Project} * @param fields List of project metadata fields + * @param locale Current users {@link Locale} * @return result message */ @Transactional - public String createMetadataFieldsForProject(Long projectId, List fields) { + public String createMetadataFieldsForProject(Long projectId, List fields, Locale locale) { Project project = projectService.read(projectId); for (ProjectMetadataField field : fields) { String label = field.getLabel(); @@ -159,7 +160,7 @@ public String createMetadataFieldsForProject(Long projectId, List = {}; - const body = headers.map((header) => ({ - label: header.name, - restriction: header.restriction, - })); - console.log("body"); - console.log(body); - await createMetadataFieldsForProject({ projectId, - body, + body: headers + .filter((header) => header.name !== sampleNameColumn) + .map((header) => ({ + label: header.name, + restriction: header.restriction, + })), }); const chunkSize = 100; From fa6f962b17a26ac8e405b2c0a18d2c748d60ddbf Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 2 Nov 2022 10:20:12 -0500 Subject: [PATCH 350/655] removing commented out code --- .../ria/web/services/UIProjectSampleService.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index eb599d11429..3f26ed1b73a 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -188,7 +188,7 @@ public ResponseEntity updateSample(UpdateSampleRequest request, Lo } /** - * Creates a metadata entry set for a sample + * Creates a metadata entry set for a sample, assuming the metadata field and restriction exist * * @param metadataFields list of {@link MetadataFieldModel}s * @param project the project the sample belongs to @@ -198,15 +198,6 @@ private Set createMetadata(List metadataField Set metadataEntrySet = metadataFields.stream().map(entry -> { String label = entry.getField(); MetadataTemplateField field = metadataTemplateService.readMetadataFieldByLabel(label); - // MetadataTemplateField field; - // MetadataTemplateField existingField = metadataTemplateService.readMetadataFieldByLabel(label); - // if (existingField != null) { - // field = existingField; - // } else { - // field = metadataTemplateService.saveMetadataField(new MetadataTemplateField(label, "text")); - // } - // ProjectMetadataRole role = ProjectMetadataRole.fromString(entry.getRestriction()); - // metadataTemplateService.setMetadataRestriction(project, field, role); return new MetadataEntry(entry.getValue(), "text", field); }).collect(Collectors.toSet()); return metadataEntrySet; From 537540a9f2a32c0b4c0b9531ac760d739c84893b Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 3 Nov 2022 13:42:40 -0500 Subject: [PATCH 351/655] storing a separate session for each project --- .../redux/importReducer.ts | 21 +++++-------------- .../samples-metadata-import/redux/store.ts | 19 ++++++++++------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts index 448d8a0e649..5a13481054a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts @@ -291,14 +291,6 @@ export const setMetadataSaveDetails = createAction( }) ); -/* -Redux action for resetting the state. -For more information on redux actions see: https://redux-toolkit.js.org/api/createAction - */ -export const resetImport = createAction(`importReducer/resetImport`, () => ({ - payload: {}, -})); - /* Redux reducer for project metadata. For more information on redux reducers see: https://redux-toolkit.js.org/api/createReducer @@ -309,6 +301,11 @@ export const importReducer = createReducer(initialState, (builder) => { }); builder.addCase(setProjectId, (state, action) => { state.projectId = action.payload.projectId; + state.sampleNameColumn = ""; + state.headers = []; + state.metadata = []; + state.metadataValidateDetails = {}; + state.metadataSaveDetails = {}; }); builder.addCase(setMetadata, (state, action) => { state.metadata = action.payload.metadata; @@ -326,12 +323,4 @@ export const importReducer = createReducer(initialState, (builder) => { builder.addCase(saveMetadata.fulfilled, (state, action) => { state.metadataSaveDetails = action.payload.metadataSaveDetails; }); - builder.addCase(resetImport, (state) => { - state.projectId = ""; - state.sampleNameColumn = ""; - state.headers = []; - state.metadata = []; - state.metadataValidateDetails = {}; - state.metadataSaveDetails = {}; - }); }); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts index cd7a6401e75..62a4d0f47d1 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/store.ts @@ -8,24 +8,29 @@ import { importReducer } from "./importReducer"; import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux"; import { getProjectIdFromUrl } from "../../../../utilities/url-utilities"; -const storageKey = "metadataImport"; +const projectId = getProjectIdFromUrl(); +const storageKey = "metadataImport-" + projectId; const storeState = (store: MiddlewareAPI) => { return (next: Dispatch) => (action: Action) => { const result = next(action); - sessionStorage.setItem(storageKey, JSON.stringify(store.getState())); + const expirationDate = new Date(new Date().getTime() + 6 * 60 * 60 * 1000); + const storage = { + state: store.getState(), + expiry: expirationDate.toISOString(), + }; + sessionStorage.setItem(storageKey, JSON.stringify(storage)); return result; }; }; const retrieveState = () => { const stringData = sessionStorage.getItem(storageKey); - if (stringData != null) { - const projectIdFromUrl = getProjectIdFromUrl(); + if (stringData !== null) { const result = JSON.parse(stringData); - const projectIdFromStorage = result.importReducer.projectId; - if (projectIdFromUrl === projectIdFromStorage) { - return result; + const expiry = new Date(result.expiry); + if (expiry > new Date()) { + return result.state; } } }; From baa77cf272972f22b7162199ca33a25bc6da360a Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 3 Nov 2022 13:52:35 -0500 Subject: [PATCH 352/655] removing redirection on completion --- .../SampleMetadataImportComplete.tsx | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx index 601562378b4..21c7df68174 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx @@ -3,14 +3,8 @@ import { useNavigate, useParams } from "react-router-dom"; import { Button, Result } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { MetadataItem } from "../../../../apis/projects/samples"; -import { - ImportDispatch, - ImportState, - useImportDispatch, - useImportSelector, -} from "../redux/store"; +import { ImportState, useImportSelector } from "../redux/store"; import { NavigateFunction } from "react-router/dist/lib/hooks"; -import { resetImport } from "../redux/importReducer"; /** * React component that displays Step #4 of the Sample Metadata Uploader. @@ -21,7 +15,6 @@ import { resetImport } from "../redux/importReducer"; export function SampleMetadataImportComplete(): JSX.Element { const { metadata, metadataValidateDetails, metadataSaveDetails } = useImportSelector((state: ImportState) => state.importReducer); - const dispatch: ImportDispatch = useImportDispatch(); const samplesUpdatedCount = metadata.filter( (metadataItem: MetadataItem) => @@ -58,14 +51,6 @@ export function SampleMetadataImportComplete(): JSX.Element { const { projectId } = useParams<{ projectId: string }>(); const navigate: NavigateFunction = useNavigate(); - const onClick = React.useCallback(async () => { - await dispatch(resetImport()); - navigate(`/${projectId}/sample-metadata/upload/file`); - }, [dispatch, projectId, navigate]); - - React.useEffect(() => { - setTimeout(onClick, 10000); - }, [onClick]); return ( @@ -74,7 +59,12 @@ export function SampleMetadataImportComplete(): JSX.Element { title={i18n("SampleMetadataImportComplete.result.title")} subTitle={stats} extra={ - } From 1eff1a33bd4146fb5198891d4eee73125bb59b22 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 3 Nov 2022 14:05:56 -0500 Subject: [PATCH 353/655] adding quick and dirty way to strip leading & trailing whitespace from file data --- .../components/SampleMetadataImportUploadFile.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx index 6f0995ca1bb..a08d52541d8 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx @@ -61,8 +61,11 @@ export function SampleMetadataImportUploadFile(): JSX.Element { rawNumbers: false, } ); - await dispatch(setHeaders({ headers: Object.keys(rows[0]) })); - await dispatch(setMetadata(rows)); + const cleanRows = JSON.parse( + JSON.stringify(rows).replace(/"\s+|\s+"/g, '"') + ); + await dispatch(setHeaders({ headers: Object.keys(cleanRows[0]) })); + await dispatch(setMetadata(cleanRows)); }; if (info.file.originFileObj) { reader.readAsBinaryString(info.file.originFileObj); From 8701f4f39a518ab29ff87a1998b83fe8416355e5 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 3 Nov 2022 16:26:38 -0500 Subject: [PATCH 354/655] fixing tests --- .../SampleMetadataImportMapHeaders.tsx | 3 ++- .../ProjectSampleMetadataImportPage.java | 27 ++++++++++--------- .../ProjectSampleMetadataImportPageIT.java | 14 ++++++---- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 7d68f77c864..9e12744359d 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -104,6 +104,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { style={{ width: 300 }} value={updatedSampleNameColumn} onChange={onSampleNameColumnChange} + className="t-metadata-uploader-sample-name-column-select" > {headers.map((header) => ( @@ -113,7 +114,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { {updatedSampleNameColumn && ( row.rowKey} columns={columns} dataSource={updatedHeaders.filter( diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectSampleMetadataImportPage.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectSampleMetadataImportPage.java index eb674a06833..d87d6d9a9f1 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectSampleMetadataImportPage.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectSampleMetadataImportPage.java @@ -24,8 +24,10 @@ public class ProjectSampleMetadataImportPage extends AbstractPage { WebElement dropzone; @FindBy(className = "t-metadata-uploader-file-button") WebElement fileBtn; - @FindBy(css = "input[type=radio]") - List headerRadios; + @FindBy(className = "t-metadata-uploader-sample-name-column-select") + WebElement sampleNameColumnSelect; + @FindBy(className = "t-metadata-uploader-headers-table") + WebElement headersTable; @FindBy(className = "t-metadata-uploader-preview-button") WebElement previewBtn; @FindBy(className = "t-metadata-uploader-upload-button") @@ -73,19 +75,18 @@ public void goToCompletePage() { wait.until(ExpectedConditions.visibilityOf(successMessage)); } - public void selectSampleNameColumn() { - headerRadios.get(3).click(); - goToReviewPage(); - } - - public String getValueForSelectedSampleNameColumn() { - for (WebElement headerRadio : headerRadios) { - boolean isSelected = headerRadio.isSelected(); - if (isSelected) { - return headerRadio.getAttribute("value"); + public void selectSampleNameColumn(String sampleNameColumn) { + sampleNameColumnSelect.click(); + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(10)); + List selectOptions = wait.until( + ExpectedConditions.presenceOfAllElementsLocatedBy(By.className("ant-select-item"))); + for (WebElement option : selectOptions) { + if (option.getAttribute("title").equals(sampleNameColumn)) { + option.click(); + return; } } - return null; + wait.until(ExpectedConditions.visibilityOf(headersTable)); } public int getUpdateCount() { diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java index 9f3e2e0e50d..88ce9a34c70 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectSampleMetadataImportPageIT.java @@ -23,6 +23,7 @@ public class ProjectSampleMetadataImportPageIT extends AbstractIridaUIITChromeDr private static final String GOOD_FILE_PATH = "src/test/resources/files/metadata-upload/good.xlsx"; private static final String MIXED_FILE_PATH = "src/test/resources/files/metadata-upload/mixed.xlsx"; private static final String INVALID_FILE_PATH = "src/test/resources/files/metadata-upload/invalid.xlsx"; + private static final String SAMPLE_NAME_COLUMN = "NLEP #"; @BeforeEach public void init() { @@ -33,7 +34,8 @@ public void init() { public void testGoodFileAndHeaders() { ProjectSampleMetadataImportPage page = ProjectSampleMetadataImportPage.goToPage(driver()); page.uploadMetadataFile(GOOD_FILE_PATH); - page.selectSampleNameColumn(); + page.selectSampleNameColumn(SAMPLE_NAME_COLUMN); + page.goToReviewPage(); assertEquals(5, page.getUpdateCount(), "Has incorrect amount of update sample rows"); assertEquals(0, page.getNewCount(), "Has incorrect amount of new sample rows"); @@ -60,7 +62,8 @@ public void testGoodFileAndHeaders() { public void testMixedFileAndHeaders() { ProjectSampleMetadataImportPage page = ProjectSampleMetadataImportPage.goToPage(driver()); page.uploadMetadataFile(MIXED_FILE_PATH); - page.selectSampleNameColumn(); + page.selectSampleNameColumn(SAMPLE_NAME_COLUMN); + page.goToReviewPage(); assertEquals(5, page.getUpdateCount(), "Has incorrect amount of update sample rows"); assertEquals(2, page.getNewCount(), "Has incorrect amount of new sample rows"); } @@ -72,7 +75,8 @@ public void testSuccessfulUpload() { // assertEquals("NLEP #", page.getValueForSelectedSampleNameColumn(), // "Has incorrect pre-populated sample name header"); // page.goToReviewPage(); - page.selectSampleNameColumn(); + page.selectSampleNameColumn(SAMPLE_NAME_COLUMN); + page.goToReviewPage(); page.goToCompletePage(); assertTrue(page.isSuccessDisplayed(), "Success message did not display"); } @@ -81,8 +85,8 @@ public void testSuccessfulUpload() { public void testFailedUpload() { ProjectSampleMetadataImportPage page = ProjectSampleMetadataImportPage.goToPage(driver()); page.uploadMetadataFile(INVALID_FILE_PATH); - page.selectSampleNameColumn(); - // page.goToReviewPage(); + page.selectSampleNameColumn(SAMPLE_NAME_COLUMN); + page.goToReviewPage(); assertTrue(page.isAlertDisplayed(), "Validation message did not display"); } } From dd723735731f749fe058e32b56be39436c8cac4d Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 4 Nov 2022 07:44:39 -0500 Subject: [PATCH 355/655] chore: Initial updates --- .../ria/web/search/SearchController.java | 53 +++---- src/main/resources/i18n/messages.properties | 2 +- src/main/webapp/entries.js | 2 +- src/main/webapp/pages/search/search.html | 88 ++---------- .../js/apis/users/{user.js => user.ts} | 0 .../resources/js/pages/search/index.tsx | 60 ++++++++ .../js/pages/search/loaders/user-loader.ts | 4 + .../resources/js/pages/search/search.js | 133 ------------------ 8 files changed, 102 insertions(+), 240 deletions(-) rename src/main/webapp/resources/js/apis/users/{user.js => user.ts} (100%) create mode 100644 src/main/webapp/resources/js/pages/search/index.tsx create mode 100644 src/main/webapp/resources/js/pages/search/loaders/user-loader.ts delete mode 100644 src/main/webapp/resources/js/pages/search/search.js diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java index 7b6a9192ec5..c41de820d2b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java @@ -1,17 +1,5 @@ package ca.corefacility.bioinformatics.irida.ria.web.search; -import java.util.List; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Sort; -import org.springframework.stereotype.Controller; -import org.springframework.ui.Model; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; - import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; @@ -23,8 +11,17 @@ import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProjectSamples; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; - import com.google.common.collect.Lists; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.List; +import java.util.stream.Collectors; /** * Controller to manage global searching @@ -40,9 +37,19 @@ public SearchController(ProjectService projectService, SampleService sampleServi this.sampleService = sampleService; } + /** + * Get the search view + * + * @return name of the search view + */ + @RequestMapping("/search") + public String search() { + return "search/search"; + } + /** * Search all projects a user is a member of based on a query string - * + * * @param query * the query string * @param global @@ -103,24 +110,6 @@ public DataTablesResponse searchSamples(@RequestParam String query, return new DataTablesResponse(params, samplePage, samples); } - /** - * Get the search view with a given query - * - * @param query the query string - * @param global Whether to perform an admin - * global search - * @param model model for the view - * @return name of the search view - */ - @RequestMapping("/search") - public String search(@RequestParam String query, - @RequestParam(required = false, defaultValue = "false") boolean global, Model model) { - model.addAttribute("searchQuery", query); - model.addAttribute("searchGlobal", global); - - return "search/search"; - } - /** * Extract the details of the a {@link Project} into a {@link DTProject} * which is consumable by the UI diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index b558738f4d0..e7a7df9d197 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2178,7 +2178,7 @@ page.cart.title=Cart # ========================================================================================== # # Search # # ========================================================================================== # -search.title=Search: {0} +search.title=Search search.samples.tab=Samples search.projects.tab=Projects search.samples.name=Sample Name diff --git a/src/main/webapp/entries.js b/src/main/webapp/entries.js index 227ebb6e480..13dc1353ca6 100644 --- a/src/main/webapp/entries.js +++ b/src/main/webapp/entries.js @@ -32,7 +32,7 @@ module.exports = { groups: "./resources/js/pages/UserGroupsPage", "project-ncbi-exports": "./resources/js/pages/projects/ncbi", "project-ncbi-export": "./resources/js/pages/projects/ncbi/create", - search: "./resources/js/pages/search/search.js", + search: "./resources/js/pages/search", user: "./resources/js/pages/user", admin: "./resources/js/pages/admin/index.tsx", "analyses-outputs": "./resources/js/pages/analyses/analyses-outputs/", diff --git a/src/main/webapp/pages/search/search.html b/src/main/webapp/pages/search/search.html index cf7519365a3..d355a349487 100644 --- a/src/main/webapp/pages/search/search.html +++ b/src/main/webapp/pages/search/search.html @@ -1,77 +1,19 @@ - - - - _Search_ - - - - - -
    - - - - - - - - - -
    -
    -
    - - - - - - - - - - - -
    -
    -
    - - -
    + + + _Search_ + + + + +
    - + - - - \ No newline at end of file + + diff --git a/src/main/webapp/resources/js/apis/users/user.js b/src/main/webapp/resources/js/apis/users/user.ts similarity index 100% rename from src/main/webapp/resources/js/apis/users/user.js rename to src/main/webapp/resources/js/apis/users/user.ts diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx new file mode 100644 index 00000000000..590fd05ea6a --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -0,0 +1,60 @@ +import { Checkbox, Input, Layout, PageHeader, Space } from "antd"; +import React, { useEffect, useRef } from "react"; +import { render } from "react-dom"; +import { + DataBrowserRouter, + Outlet, + Route, + Routes, + useLoaderData, + useSearchParams, +} from "react-router-dom"; +import { setBaseUrl } from "../../utilities/url-utilities"; +import userLoader from "./loaders/user-loader"; + +function SearchPage() { + return ( + + + } /> + + + ); +} + +function SearchLayout() { + const user = useLoaderData(); + console.log(user); + const [searchParams, setSearchParams] = useSearchParams(); + const query = useRef(searchParams.get("query")); + console.log(query); + + return ( + + + + +
    + Project + Samples + Search Global +
    +     +     +     + ); +} + +const element = document.querySelector("#root"); +render( + + + } + loader={userLoader} + /> + + , + element +); diff --git a/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts b/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts new file mode 100644 index 00000000000..48802c940cd --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts @@ -0,0 +1,4 @@ +import { fetchCurrentUserDetails } from "../../../apis/users/user"; +export async function loader() { + return fetchCurrentUserDetails(); +} diff --git a/src/main/webapp/resources/js/pages/search/search.js b/src/main/webapp/resources/js/pages/search/search.js deleted file mode 100644 index ca1188118ff..00000000000 --- a/src/main/webapp/resources/js/pages/search/search.js +++ /dev/null @@ -1,133 +0,0 @@ -import "../../vendor/datatables/datatables"; -import React from "react"; -import { Button } from "antd"; -import $ from "jquery"; -import { - createItemLink, - generateColumnOrderInfo, - tableConfig, - wrapCellContents, -} from "./../../utilities/datatables-utilities"; -import { formatDate } from "./../../utilities/date-utilities"; -import { SampleDetailViewer } from "../../components/samples/SampleDetailViewer"; -import * as ReactDOM from "react-dom"; - -/* -Get the table headers and create a look up table for them. -This give the row name in snake case and its index. - */ -const PROJECT_COLUMNS = generateColumnOrderInfo("#projects"); -const SAMPLE_COLUMNS = generateColumnOrderInfo("#samples"); - -const projectConfig = Object.assign({}, tableConfig, { - ajax: window.PAGE.urls.projects, - searching: false, - order: [[PROJECT_COLUMNS.MODIFIED_DATE, "desc"]], - initComplete: function (settings, json) { - $("#project-count").text(json.recordsTotal); - }, - columnDefs: [ - { - targets: [PROJECT_COLUMNS.NAME], - render(data, type, full) { - // Render the name as a link to the actual project. - return createItemLink({ - url: `${window.PAGE.urls.project}${full.id}`, - label: `${ - full.remote - ? `` - : data - }`, - width: "200px", - }); - }, - }, - { - targets: PROJECT_COLUMNS.ORGANISM, - render(data) { - return wrapCellContents({ text: data }); - }, - }, - // Format all dates to standate date for the systme. - { - targets: [PROJECT_COLUMNS.CREATED_DATE, PROJECT_COLUMNS.MODIFIED_DATE], - render(data) { - const date = formatDate({ date: data }); - return ``; - }, - }, - ], -}); - -const sampleConfig = Object.assign({}, tableConfig, { - ajax: window.PAGE.urls.samples, - searching: false, - order: [[SAMPLE_COLUMNS.MODIFIED_DATE, "desc"]], - initComplete: function (settings, json) { - $("#sample-count").text(json.recordsTotal); - }, - columnDefs: [ - { - targets: [SAMPLE_COLUMNS.SAMPLE_NAME], - createdCell: (td, cellData, rowData) => { - ReactDOM.render( - - - , - td - ); - }, - }, - { - targets: SAMPLE_COLUMNS.ORGANISM, - render(data) { - return wrapCellContents({ text: data }); - }, - }, - { - targets: [SAMPLE_COLUMNS.PROJECT_NAME], - render(data, type, full) { - // Render the name as a link to the actual project. - return createItemLink({ - url: `${window.PAGE.urls.project}${full.projectId}`, - label: data, - }); - }, - }, - // Format all dates to standate date for the systme. - { - targets: [SAMPLE_COLUMNS.CREATED_DATE, SAMPLE_COLUMNS.MODIFIED_DATE], - render(data) { - const date = formatDate({ date: data }); - return ``; - }, - }, - ], -}); - -// init the datatables -$("#projects").DataTable(projectConfig); -$("#samples").DataTable(sampleConfig); - -// update the page hash on tab click -$('a[data-toggle="tab"]').on("shown.bs.tab", function (e) { - var hash = $(this).attr("href"); - window.location.hash = hash; -}); - -$(document).ready(function () { - var hash = window.location.hash; - - if (hash === "#project-tab") { - $('.nav-tabs a[href="#project-tab"]').tab("show"); - } else if (hash === "#sample-tab") { - $('.nav-tabs a[href="#sample-tab"]').tab("show"); - } -}); From aa98a61f354662ed98fd6fbc077aeb7876365a5d Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 4 Nov 2022 08:16:42 -0500 Subject: [PATCH 356/655] fix: updated ProjectSPA.tsx to use new react-router syntax --- src/main/webapp/package.json | 15 +- src/main/webapp/pnpm-lock.yaml | 1685 +++++++++-------- .../js/pages/projects/ProjectSPA.tsx | 73 +- 3 files changed, 934 insertions(+), 839 deletions(-) diff --git a/src/main/webapp/package.json b/src/main/webapp/package.json index a7ff2001b73..5cf7caf5941 100644 --- a/src/main/webapp/package.json +++ b/src/main/webapp/package.json @@ -26,7 +26,6 @@ "@antv/path-util": "^2.0.15", "@antv/scale": "^0.3.17", "@antv/util": "^2.0.17", - "@babel/preset-typescript": "^7.17.12", "@fortawesome/fontawesome-free": "^6.1.1", "@loaders.gl/core": ">=3.2.0 <4.0.0", "@loaders.gl/gltf": "^3.0.0", @@ -37,7 +36,6 @@ "@luma.gl/webgl": ">=8.4.0 <9.0.0", "@phylocanvas/phylocanvas.gl": "^1.43.0", "@reduxjs/toolkit": "1.8.5", - "@remix-run/router": "^0.1.0", "ag-grid-community": "^27.1.0", "ag-grid-react": "^27.1.0", "antd": "4.19.5", @@ -75,7 +73,7 @@ "react-markdown": "^8.0.2", "react-mde": "^11.5.0", "react-redux": "^7.2.8", - "react-router-dom": "^6.4.0-pre.7", + "react-router-dom": "^6.4.3", "react-virtualized-auto-sizer": "^1.0.6", "react-window": "^1.8.6", "reactour": "^1.18.7", @@ -88,11 +86,12 @@ "xlsx": "^0.18.5" }, "devDependencies": { - "@babel/core": "^7.17.9", - "@babel/plugin-proposal-export-default-from": "^7.16.7", - "@babel/preset-env": "^7.16.11", - "@babel/preset-react": "^7.16.7", - "@babel/runtime": "^7.17.9", + "@babel/core": "^7.19.6", + "@babel/plugin-proposal-export-default-from": "^7.18.10", + "@babel/preset-env": "^7.19.4", + "@babel/preset-react": "^7.18.6", + "@babel/preset-typescript": "^7.18.6", + "@babel/runtime": "^7.20.1", "@types/lodash": "^4.14.182", "@types/lodash.uniqby": "^4.7.7", "@types/node": "^17.0.39", diff --git a/src/main/webapp/pnpm-lock.yaml b/src/main/webapp/pnpm-lock.yaml index 0e7d54ac308..eac4699038b 100644 --- a/src/main/webapp/pnpm-lock.yaml +++ b/src/main/webapp/pnpm-lock.yaml @@ -19,12 +19,12 @@ specifiers: '@antv/path-util': ^2.0.15 '@antv/scale': ^0.3.17 '@antv/util': ^2.0.17 - '@babel/core': ^7.17.9 - '@babel/plugin-proposal-export-default-from': ^7.16.7 - '@babel/preset-env': ^7.16.11 - '@babel/preset-react': ^7.16.7 - '@babel/preset-typescript': ^7.17.12 - '@babel/runtime': ^7.17.9 + '@babel/core': ^7.19.6 + '@babel/plugin-proposal-export-default-from': ^7.18.10 + '@babel/preset-env': ^7.19.4 + '@babel/preset-react': ^7.18.6 + '@babel/preset-typescript': ^7.18.6 + '@babel/runtime': ^7.20.1 '@fortawesome/fontawesome-free': ^6.1.1 '@loaders.gl/core': '>=3.2.0 <4.0.0' '@loaders.gl/gltf': ^3.0.0 @@ -35,7 +35,6 @@ specifiers: '@luma.gl/webgl': '>=8.4.0 <9.0.0' '@phylocanvas/phylocanvas.gl': ^1.43.0 '@reduxjs/toolkit': 1.8.5 - '@remix-run/router': ^0.1.0 '@types/lodash': ^4.14.182 '@types/lodash.uniqby': ^4.7.7 '@types/node': ^17.0.39 @@ -108,7 +107,7 @@ specifiers: react-markdown: ^8.0.2 react-mde: ^11.5.0 react-redux: ^7.2.8 - react-router-dom: ^6.4.0-pre.7 + react-router-dom: ^6.4.3 react-virtualized-auto-sizer: ^1.0.6 react-window: ^1.8.6 reactour: ^1.18.7 @@ -135,7 +134,6 @@ dependencies: '@antv/path-util': 2.0.15 '@antv/scale': 0.3.17 '@antv/util': 2.0.17 - '@babel/preset-typescript': 7.17.12_@babel+core@7.17.9 '@fortawesome/fontawesome-free': 6.1.1 '@loaders.gl/core': 3.2.4 '@loaders.gl/gltf': 3.2.4 @@ -146,7 +144,6 @@ dependencies: '@luma.gl/webgl': 8.5.14 '@phylocanvas/phylocanvas.gl': 1.43.0_deck.gl@8.7.12 '@reduxjs/toolkit': 1.8.5_bjryulbxll5jujtwpu5a2wm2cy - '@remix-run/router': 0.1.0 ag-grid-community: 27.1.0 ag-grid-react: 27.1.0_w6hrqnyrd37qujkkojqwgzxsou antd: 4.19.5_sfoxds7t5ydpegc3knd667wn6m @@ -184,7 +181,7 @@ dependencies: react-markdown: 8.0.2_hx2b44akkvgcgvvtmk7ds2qk6q react-mde: 11.5.0_sfoxds7t5ydpegc3knd667wn6m react-redux: 7.2.8_sfoxds7t5ydpegc3knd667wn6m - react-router-dom: 6.4.0-pre.8_sfoxds7t5ydpegc3knd667wn6m + react-router-dom: 6.4.3_sfoxds7t5ydpegc3knd667wn6m react-virtualized-auto-sizer: 1.0.6_sfoxds7t5ydpegc3knd667wn6m react-window: 1.8.6_sfoxds7t5ydpegc3knd667wn6m reactour: 1.18.7_c7dgqabs4tvn3myjgibvwpzy7e @@ -197,11 +194,12 @@ dependencies: xlsx: 0.18.5 devDependencies: - '@babel/core': 7.17.9 - '@babel/plugin-proposal-export-default-from': 7.16.7_@babel+core@7.17.9 - '@babel/preset-env': 7.16.11_@babel+core@7.17.9 - '@babel/preset-react': 7.16.7_@babel+core@7.17.9 - '@babel/runtime': 7.17.9 + '@babel/core': 7.19.6 + '@babel/plugin-proposal-export-default-from': 7.18.10_@babel+core@7.19.6 + '@babel/preset-env': 7.19.4_@babel+core@7.19.6 + '@babel/preset-react': 7.18.6_@babel+core@7.19.6 + '@babel/preset-typescript': 7.18.6_@babel+core@7.19.6 + '@babel/runtime': 7.20.1 '@types/lodash': 4.14.182 '@types/lodash.uniqby': 4.7.7 '@types/node': 17.0.39 @@ -214,7 +212,7 @@ devDependencies: '@typescript-eslint/eslint-plugin': 5.27.0_ljfpvv7dypng7yyhsvie2ogdeu '@typescript-eslint/parser': 5.27.0_6hvhxfirl4y3p6nuemvvvbadcu autoprefixer: 10.4.4_postcss@8.4.12 - babel-loader: 8.2.4_vs5hf2sl7hjtttp43d2vzw3qay + babel-loader: 8.2.4_h3kngfo6sqnotbut4nntjyiezm babel-plugin-import: 1.13.5 browserslist: 4.20.2 css-loader: 6.7.1_webpack@5.72.0 @@ -261,6 +259,7 @@ packages: engines: {node: '>=6.0.0'} dependencies: '@jridgewell/trace-mapping': 0.3.2 + dev: true /@ant-design/colors/6.0.0: resolution: {integrity: sha512-qAZRvPzfdWHtfameEGP2Qvuf838NhergR35o+EuVyB5XvSA98xod5r4utvi4TJ3ywmevm290g9nsCG5MryrdWQ==} @@ -281,7 +280,7 @@ packages: dependencies: '@ant-design/colors': 6.0.0 '@ant-design/icons-svg': 4.2.1 - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -305,7 +304,7 @@ packages: peerDependencies: react: '>=16.0.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 json2mq: 0.2.0 lodash: 4.17.21 @@ -481,30 +480,31 @@ packages: tslib: 2.3.1 dev: false - /@babel/code-frame/7.16.7: - resolution: {integrity: sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==} + /@babel/code-frame/7.18.6: + resolution: {integrity: sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==} engines: {node: '>=6.9.0'} dependencies: - '@babel/highlight': 7.16.10 + '@babel/highlight': 7.18.6 - /@babel/compat-data/7.17.7: - resolution: {integrity: sha512-p8pdE6j0a29TNGebNm7NzYZWB3xVZJBZ7XGs42uAKzQo8VQ3F0By/cQCtUEABwIqw5zo6WA4NbmxsfzADzMKnQ==} + /@babel/compat-data/7.20.1: + resolution: {integrity: sha512-EWZ4mE2diW3QALKvDMiXnbZpRvlj+nayZ112nK93SnhqOtpdsbVD4W+2tEoT3YNBAG9RBR0ISY758ZkOgsn6pQ==} engines: {node: '>=6.9.0'} + dev: true - /@babel/core/7.17.9: - resolution: {integrity: sha512-5ug+SfZCpDAkVp9SFIZAzlW18rlzsOcJGaetCjkySnrXXDUw9AR8cDUm1iByTmdWM6yxX6/zycaV76w3YTF2gw==} + /@babel/core/7.19.6: + resolution: {integrity: sha512-D2Ue4KHpc6Ys2+AxpIx1BZ8+UegLLLE2p3KJEuJRKmokHOtl49jQ5ny1773KsGLZs8MQvBidAF6yWUJxRqtKtg==} engines: {node: '>=6.9.0'} dependencies: '@ampproject/remapping': 2.1.2 - '@babel/code-frame': 7.16.7 - '@babel/generator': 7.17.9 - '@babel/helper-compilation-targets': 7.17.7_@babel+core@7.17.9 - '@babel/helper-module-transforms': 7.17.7 - '@babel/helpers': 7.17.9 - '@babel/parser': 7.17.9 - '@babel/template': 7.16.7 - '@babel/traverse': 7.17.9 - '@babel/types': 7.17.0 + '@babel/code-frame': 7.18.6 + '@babel/generator': 7.20.1 + '@babel/helper-compilation-targets': 7.20.0_@babel+core@7.19.6 + '@babel/helper-module-transforms': 7.19.6 + '@babel/helpers': 7.20.1 + '@babel/parser': 7.20.1 + '@babel/template': 7.18.10 + '@babel/traverse': 7.20.1 + '@babel/types': 7.20.0 convert-source-map: 1.7.0 debug: 4.3.4 gensync: 1.0.0-beta.2 @@ -512,79 +512,80 @@ packages: semver: 6.3.0 transitivePeerDependencies: - supports-color + dev: true - /@babel/generator/7.17.9: - resolution: {integrity: sha512-rAdDousTwxbIxbz5I7GEQ3lUip+xVCXooZNbsydCWs3xA7ZsYOv+CFRdzGxRX78BmQHu9B1Eso59AOZQOJDEdQ==} + /@babel/generator/7.20.1: + resolution: {integrity: sha512-u1dMdBUmA7Z0rBB97xh8pIhviK7oItYOkjbsCxTWMknyvbQRBwX7/gn4JXurRdirWMFh+ZtYARqkA6ydogVZpg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 + '@jridgewell/gen-mapping': 0.3.2 jsesc: 2.5.2 - source-map: 0.5.7 - /@babel/helper-annotate-as-pure/7.16.7: - resolution: {integrity: sha512-s6t2w/IPQVTAET1HitoowRGXooX8mCgtuP5195wD/QJPV6wYjpujCGF7JuMODVX2ZAJOf1GT6DT9MHEZvLOFSw==} + /@babel/helper-annotate-as-pure/7.18.6: + resolution: {integrity: sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 - /@babel/helper-builder-binary-assignment-operator-visitor/7.16.7: - resolution: {integrity: sha512-C6FdbRaxYjwVu/geKW4ZeQ0Q31AftgRcdSnZ5/jsH6BzCJbtvXvhpfkbkThYSuutZA7nCXpPR6AD9zd1dprMkA==} + /@babel/helper-builder-binary-assignment-operator-visitor/7.18.9: + resolution: {integrity: sha512-yFQ0YCHoIqarl8BCRwBL8ulYUaZpz3bNsA7oFepAzee+8/+ImtADXNOmO5vJvsPff3qi+hvpkY/NYBTrBQgdNw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-explode-assignable-expression': 7.16.7 - '@babel/types': 7.17.0 + '@babel/helper-explode-assignable-expression': 7.18.6 + '@babel/types': 7.20.0 dev: true - /@babel/helper-compilation-targets/7.17.7_@babel+core@7.17.9: - resolution: {integrity: sha512-UFzlz2jjd8kroj0hmCFV5zr+tQPi1dpC2cRsDV/3IEW8bJfCPrPpmcSN6ZS8RqIq4LXcmpipCQFPddyFA5Yc7w==} + /@babel/helper-compilation-targets/7.20.0_@babel+core@7.19.6: + resolution: {integrity: sha512-0jp//vDGp9e8hZzBc6N/KwA5ZK3Wsm/pfm4CrY7vzegkVxc65SgSn6wYOnwHe9Js9HRQ1YTCKLGPzDtaS3RoLQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/compat-data': 7.17.7 - '@babel/core': 7.17.9 - '@babel/helper-validator-option': 7.16.7 - browserslist: 4.20.2 + '@babel/compat-data': 7.20.1 + '@babel/core': 7.19.6 + '@babel/helper-validator-option': 7.18.6 + browserslist: 4.21.4 semver: 6.3.0 + dev: true - /@babel/helper-create-class-features-plugin/7.18.0_@babel+core@7.17.9: - resolution: {integrity: sha512-Kh8zTGR9de3J63e5nS0rQUdRs/kbtwoeQQ0sriS0lItjC96u8XXZN6lKpuyWd2coKSU13py/y+LTmThLuVX0Pg==} + /@babel/helper-create-class-features-plugin/7.19.0_@babel+core@7.19.6: + resolution: {integrity: sha512-NRz8DwF4jT3UfrmUoZjd0Uph9HQnP30t7Ash+weACcyNkiYTywpIjDBgReJMKgr+n86sn2nPVVmJ28Dm053Kqw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-environment-visitor': 7.16.7 - '@babel/helper-function-name': 7.17.9 - '@babel/helper-member-expression-to-functions': 7.17.7 - '@babel/helper-optimise-call-expression': 7.16.7 - '@babel/helper-replace-supers': 7.16.7 - '@babel/helper-split-export-declaration': 7.16.7 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-function-name': 7.19.0 + '@babel/helper-member-expression-to-functions': 7.18.9 + '@babel/helper-optimise-call-expression': 7.18.6 + '@babel/helper-replace-supers': 7.19.1 + '@babel/helper-split-export-declaration': 7.18.6 transitivePeerDependencies: - supports-color + dev: true - /@babel/helper-create-regexp-features-plugin/7.17.0_@babel+core@7.17.9: - resolution: {integrity: sha512-awO2So99wG6KnlE+TPs6rn83gCz5WlEePJDTnLEqbchMVrBeAujURVphRdigsk094VhvZehFoNOihSlcBjwsXA==} + /@babel/helper-create-regexp-features-plugin/7.19.0_@babel+core@7.19.6: + resolution: {integrity: sha512-htnV+mHX32DF81amCDrwIDr8nrp1PTm+3wfBN9/v8QJOLEioOCOG7qNyq0nHeFiWbT3Eb7gsPwEmV64UCQ1jzw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-annotate-as-pure': 7.16.7 - regexpu-core: 5.0.1 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + regexpu-core: 5.2.1 dev: true - /@babel/helper-define-polyfill-provider/0.3.1_@babel+core@7.17.9: - resolution: {integrity: sha512-J9hGMpJQmtWmj46B3kBHmL38UhJGhYX7eqkcq+2gsstyYt341HmPeWspihX43yVRA0mS+8GGk2Gckc7bY/HCmA==} + /@babel/helper-define-polyfill-provider/0.3.3_@babel+core@7.19.6: + resolution: {integrity: sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==} peerDependencies: '@babel/core': ^7.4.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-compilation-targets': 7.17.7_@babel+core@7.17.9 - '@babel/helper-module-imports': 7.16.7 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/traverse': 7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-compilation-targets': 7.20.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 debug: 4.3.4 lodash.debounce: 4.0.8 resolve: 1.22.0 @@ -593,1155 +594,1190 @@ packages: - supports-color dev: true - /@babel/helper-environment-visitor/7.16.7: - resolution: {integrity: sha512-SLLb0AAn6PkUeAfKJCCOl9e1R53pQlGAfc4y4XuMRZfqeMYLE0dM1LMhqbGAlGQY0lfw5/ohoYWAe9V1yibRag==} + /@babel/helper-environment-visitor/7.18.9: + resolution: {integrity: sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==} engines: {node: '>=6.9.0'} - dependencies: - '@babel/types': 7.17.0 - /@babel/helper-explode-assignable-expression/7.16.7: - resolution: {integrity: sha512-KyUenhWMC8VrxzkGP0Jizjo4/Zx+1nNZhgocs+gLzyZyB8SHidhoq9KK/8Ato4anhwsivfkBLftky7gvzbZMtQ==} + /@babel/helper-explode-assignable-expression/7.18.6: + resolution: {integrity: sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 dev: true - /@babel/helper-function-name/7.17.9: - resolution: {integrity: sha512-7cRisGlVtiVqZ0MW0/yFB4atgpGLWEHUVYnb448hZK4x+vih0YO5UoS11XIYtZYqHd0dIPMdUSv8q5K4LdMnIg==} + /@babel/helper-function-name/7.19.0: + resolution: {integrity: sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w==} engines: {node: '>=6.9.0'} dependencies: - '@babel/template': 7.16.7 - '@babel/types': 7.17.0 + '@babel/template': 7.18.10 + '@babel/types': 7.20.0 - /@babel/helper-hoist-variables/7.16.7: - resolution: {integrity: sha512-m04d/0Op34H5v7pbZw6pSKP7weA6lsMvfiIAMeIvkY/R4xQtBSMFEigu9QTZ2qB/9l22vsxtM8a+Q8CzD255fg==} + /@babel/helper-hoist-variables/7.18.6: + resolution: {integrity: sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 - /@babel/helper-member-expression-to-functions/7.17.7: - resolution: {integrity: sha512-thxXgnQ8qQ11W2wVUObIqDL4p148VMxkt5T/qpN5k2fboRyzFGFmKsTGViquyM5QHKUy48OZoca8kw4ajaDPyw==} + /@babel/helper-member-expression-to-functions/7.18.9: + resolution: {integrity: sha512-RxifAh2ZoVU67PyKIO4AMi1wTenGfMR/O/ae0CCRqwgBAt5v7xjdtRw7UoSbsreKrQn5t7r89eruK/9JjYHuDg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 + dev: true /@babel/helper-module-imports/7.16.7: resolution: {integrity: sha512-LVtS6TqjJHFc+nYeITRo6VLXve70xmq7wPhWTqDJusJEgGmkAACWwMiTNrvfoQo6hEhFwAIixNkvB0jPXDL8Wg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 - /@babel/helper-module-transforms/7.17.7: - resolution: {integrity: sha512-VmZD99F3gNTYB7fJRDTi+u6l/zxY0BE6OIxPSU7a50s6ZUQkHwSDmV92FfM+oCG0pZRVojGYhkR8I0OGeCVREw==} + /@babel/helper-module-imports/7.18.6: + resolution: {integrity: sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-environment-visitor': 7.16.7 - '@babel/helper-module-imports': 7.16.7 - '@babel/helper-simple-access': 7.17.7 - '@babel/helper-split-export-declaration': 7.16.7 - '@babel/helper-validator-identifier': 7.16.7 - '@babel/template': 7.16.7 - '@babel/traverse': 7.17.9 - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 + + /@babel/helper-module-transforms/7.19.6: + resolution: {integrity: sha512-fCmcfQo/KYr/VXXDIyd3CBGZ6AFhPFy1TfSEJ+PilGVlQT6jcbqtHAM4C1EciRqMza7/TpOUZliuSH+U6HAhJw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-module-imports': 7.18.6 + '@babel/helper-simple-access': 7.19.4 + '@babel/helper-split-export-declaration': 7.18.6 + '@babel/helper-validator-identifier': 7.19.1 + '@babel/template': 7.18.10 + '@babel/traverse': 7.20.1 + '@babel/types': 7.20.0 transitivePeerDependencies: - supports-color + dev: true - /@babel/helper-optimise-call-expression/7.16.7: - resolution: {integrity: sha512-EtgBhg7rd/JcnpZFXpBy0ze1YRfdm7BnBX4uKMBd3ixa3RGAE002JZB66FJyNH7g0F38U05pXmA5P8cBh7z+1w==} + /@babel/helper-optimise-call-expression/7.18.6: + resolution: {integrity: sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 + dev: true - /@babel/helper-plugin-utils/7.17.12: - resolution: {integrity: sha512-JDkf04mqtN3y4iAbO1hv9U2ARpPyPL1zqyWs/2WG1pgSq9llHFjStX5jdxb84himgJm+8Ng+x0oiWF/nw/XQKA==} + /@babel/helper-plugin-utils/7.19.0: + resolution: {integrity: sha512-40Ryx7I8mT+0gaNxm8JGTZFUITNqdLAgdg0hXzeVZxVD6nFsdhQvip6v8dqkRHzsz1VFpFAaOCHNn0vKBL7Czw==} engines: {node: '>=6.9.0'} + dev: true - /@babel/helper-remap-async-to-generator/7.16.8: - resolution: {integrity: sha512-fm0gH7Flb8H51LqJHy3HJ3wnE1+qtYR2A99K06ahwrawLdOFsCEWjZOrYricXJHoPSudNKxrMBUPEIPxiIIvBw==} + /@babel/helper-remap-async-to-generator/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-dI7q50YKd8BAv3VEfgg7PS7yD3Rtbi2J1XMXaalXO0W0164hYLnh8zpjRS0mte9MfVp/tltvr/cfdXPvJr1opA==} engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 dependencies: - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-wrap-function': 7.16.8 - '@babel/types': 7.17.0 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-wrap-function': 7.19.0 + '@babel/types': 7.20.0 transitivePeerDependencies: - supports-color dev: true - /@babel/helper-replace-supers/7.16.7: - resolution: {integrity: sha512-y9vsWilTNaVnVh6xiJfABzsNpgDPKev9HnAgz6Gb1p6UUwf9NepdlsV7VXGCftJM+jqD5f7JIEubcpLjZj5dBw==} + /@babel/helper-replace-supers/7.19.1: + resolution: {integrity: sha512-T7ahH7wV0Hfs46SFh5Jz3s0B6+o8g3c+7TMxu7xKfmHikg7EAZ3I2Qk9LFhjxXq8sL7UkP5JflezNwoZa8WvWw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-environment-visitor': 7.16.7 - '@babel/helper-member-expression-to-functions': 7.17.7 - '@babel/helper-optimise-call-expression': 7.16.7 - '@babel/traverse': 7.17.9 - '@babel/types': 7.17.0 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-member-expression-to-functions': 7.18.9 + '@babel/helper-optimise-call-expression': 7.18.6 + '@babel/traverse': 7.20.1 + '@babel/types': 7.20.0 transitivePeerDependencies: - supports-color + dev: true - /@babel/helper-simple-access/7.17.7: - resolution: {integrity: sha512-txyMCGroZ96i+Pxr3Je3lzEJjqwaRC9buMUgtomcrLe5Nd0+fk1h0LLA+ixUF5OW7AhHuQ7Es1WcQJZmZsz2XA==} + /@babel/helper-simple-access/7.19.4: + resolution: {integrity: sha512-f9Xq6WqBFqaDfbCzn2w85hwklswz5qsKlh7f08w4Y9yhJHpnNC0QemtSkK5YyOY8kPGvyiwdzZksGUhnGdaUIg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 + dev: true - /@babel/helper-skip-transparent-expression-wrappers/7.16.0: - resolution: {integrity: sha512-+il1gTy0oHwUsBQZyJvukbB4vPMdcYBrFHa0Uc4AizLxbq6BOYC51Rv4tWocX9BLBDLZ4kc6qUFpQ6HRgL+3zw==} + /@babel/helper-skip-transparent-expression-wrappers/7.20.0: + resolution: {integrity: sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 dev: true - /@babel/helper-split-export-declaration/7.16.7: - resolution: {integrity: sha512-xbWoy/PFoxSWazIToT9Sif+jJTlrMcndIsaOKvTA6u7QEo7ilkRZpjew18/W3c7nm8fXdUDXh02VXTbZ0pGDNw==} + /@babel/helper-split-export-declaration/7.18.6: + resolution: {integrity: sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 - /@babel/helper-validator-identifier/7.16.7: - resolution: {integrity: sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw==} + /@babel/helper-string-parser/7.19.4: + resolution: {integrity: sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==} engines: {node: '>=6.9.0'} - /@babel/helper-validator-option/7.16.7: - resolution: {integrity: sha512-TRtenOuRUVo9oIQGPC5G9DgK4743cdxvtOw0weQNpZXaS16SCBi5MNjZF8vba3ETURjZpTbVn7Vvcf2eAwFozQ==} + /@babel/helper-validator-identifier/7.19.1: + resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} engines: {node: '>=6.9.0'} - /@babel/helper-wrap-function/7.16.8: - resolution: {integrity: sha512-8RpyRVIAW1RcDDGTA+GpPAwV22wXCfKOoM9bet6TLkGIFTkRQSkH1nMQ5Yet4MpoXe1ZwHPVtNasc2w0uZMqnw==} + /@babel/helper-validator-option/7.18.6: + resolution: {integrity: sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==} + engines: {node: '>=6.9.0'} + dev: true + + /@babel/helper-wrap-function/7.19.0: + resolution: {integrity: sha512-txX8aN8CZyYGTwcLhlk87KRqncAzhh5TpQamZUa0/u3an36NtDpUP6bQgBCBcLeBs09R/OwQu3OjK0k/HwfNDg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-function-name': 7.17.9 - '@babel/template': 7.16.7 - '@babel/traverse': 7.17.9 - '@babel/types': 7.17.0 + '@babel/helper-function-name': 7.19.0 + '@babel/template': 7.18.10 + '@babel/traverse': 7.20.1 + '@babel/types': 7.20.0 transitivePeerDependencies: - supports-color dev: true - /@babel/helpers/7.17.9: - resolution: {integrity: sha512-cPCt915ShDWUEzEp3+UNRktO2n6v49l5RSnG9M5pS24hA+2FAc5si+Pn1i4VVbQQ+jh+bIZhPFQOJOzbrOYY1Q==} + /@babel/helpers/7.20.1: + resolution: {integrity: sha512-J77mUVaDTUJFZ5BpP6mMn6OIl3rEWymk2ZxDBQJUG3P+PbmyMcF3bYWvz0ma69Af1oobDqT/iAsvzhB58xhQUg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/template': 7.16.7 - '@babel/traverse': 7.17.9 - '@babel/types': 7.17.0 + '@babel/template': 7.18.10 + '@babel/traverse': 7.20.1 + '@babel/types': 7.20.0 transitivePeerDependencies: - supports-color + dev: true - /@babel/highlight/7.16.10: - resolution: {integrity: sha512-5FnTQLSLswEj6IkgVw5KusNUUFY9ZGqe/TRFnP/BKYHYgfh7tc+C7mwiy95/yNP7Dh9x580Vv8r7u7ZfTBFxdw==} + /@babel/highlight/7.18.6: + resolution: {integrity: sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-validator-identifier': 7.16.7 + '@babel/helper-validator-identifier': 7.19.1 chalk: 2.4.2 js-tokens: 4.0.0 - /@babel/parser/7.17.9: - resolution: {integrity: sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg==} + /@babel/parser/7.20.1: + resolution: {integrity: sha512-hp0AYxaZJhxULfM1zyp7Wgr+pSUKBcP3M+PHnSzWGdXOzg/kHWIgiUWARvubhUKGOEw3xqY4x+lyZ9ytBVcELw==} engines: {node: '>=6.0.0'} hasBin: true dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 - /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-anv/DObl7waiGEnC24O9zqL0pSuI9hljihqiDuFHC8d7/bjr/4RLGPWuc8rYOff/QPzbEPSkzG8wGG9aDuhHRg==} + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-di8vUHRdf+4aJ7ltXhaDbPoszdkh59AQtJM5soLsuHpQJdFQZOA4uGj0V2u/CZ8bJ/u8ULDL5yq6FO/bCXnKHw==} + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-AHrP9jadvH7qlOj6PINbgSuphjQUAK7AOT7DPjBo9EHoLhQTnnK5u45e1Hd4DbSQEO9nqPWtQ89r+XEOWFScKg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.13.0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-skip-transparent-expression-wrappers': 7.16.0 - '@babel/plugin-proposal-optional-chaining': 7.16.7_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 + '@babel/plugin-proposal-optional-chaining': 7.18.9_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-async-generator-functions/7.16.8_@babel+core@7.17.9: - resolution: {integrity: sha512-71YHIvMuiuqWJQkebWJtdhQTfd4Q4mF76q2IX37uZPkG9+olBxsX+rH1vkhFto4UeJZ9dPY2s+mDvhDm1u2BGQ==} + /@babel/plugin-proposal-async-generator-functions/7.20.1_@babel+core@7.19.6: + resolution: {integrity: sha512-Gh5rchzSwE4kC+o/6T8waD0WHEQIsDmjltY8WnWRXHUdH8axZhuH86Ov9M72YhJfDrZseQwuuWaaIT/TmePp3g==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-remap-async-to-generator': 7.16.8 - '@babel/plugin-syntax-async-generators': 7.8.4_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-remap-async-to-generator': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-syntax-async-generators': 7.8.4_@babel+core@7.19.6 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-proposal-class-properties/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-IobU0Xme31ewjYOShSIqd/ZGM/r/cuOz2z0MDbNrhF5FW+ZVgi0f2lyeoj9KFPDOAqsYxmLWZte1WOwlvY9aww==} + /@babel/plugin-proposal-class-properties/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-class-features-plugin': 7.18.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-create-class-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-proposal-class-static-block/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-dgqJJrcZoG/4CkMopzhPJjGxsIe9A8RlkQLnL/Vhhx8AA9ZuaRwGSlscSh42hazc7WSrya/IK7mTeoF0DP9tEw==} + /@babel/plugin-proposal-class-static-block/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.12.0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-class-features-plugin': 7.18.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-class-static-block': 7.14.5_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-create-class-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-class-static-block': 7.14.5_@babel+core@7.19.6 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-proposal-dynamic-import/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-I8SW9Ho3/8DRSdmDdH3gORdyUuYnk1m4cMxUAdu5oy4n3OfN8flDEH+d60iG7dUfi0KkYwSvoalHzzdRzpWHTg==} + /@babel/plugin-proposal-dynamic-import/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-dynamic-import': 7.8.3_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-dynamic-import': 7.8.3_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-export-default-from/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-+cENpW1rgIjExn+o5c8Jw/4BuH4eGKKYvkMB8/0ZxFQ9mC0t4z09VsPIwNg6waF69QYC81zxGeAsREGuqQoKeg==} + /@babel/plugin-proposal-export-default-from/7.18.10_@babel+core@7.19.6: + resolution: {integrity: sha512-5H2N3R2aQFxkV4PIBUR/i7PUSwgTZjouJKzI8eKswfIjT0PhvzkPn0t0wIS5zn6maQuvtT0t1oHtMUz61LOuow==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-export-default-from': 7.16.7_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-export-default-from': 7.18.6_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-export-namespace-from/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-ZxdtqDXLRGBL64ocZcs7ovt71L3jhC1RGSyR996svrCi3PYqHNkb3SwPJCs8RIzD86s+WPpt2S73+EHCGO+NUA==} + /@babel/plugin-proposal-export-namespace-from/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-export-namespace-from': 7.8.3_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-export-namespace-from': 7.8.3_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-json-strings/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-lNZ3EEggsGY78JavgbHsK9u5P3pQaW7k4axlgFLYkMd7UBsiNahCITShLjNQschPyjtO6dADrL24757IdhBrsQ==} + /@babel/plugin-proposal-json-strings/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-json-strings': 7.8.3_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-json-strings': 7.8.3_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-logical-assignment-operators/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-K3XzyZJGQCr00+EtYtrDjmwX7o7PLK6U9bi1nCwkQioRFVUv6dJoxbQjtWVtP+bCPy82bONBKG8NPyQ4+i6yjg==} + /@babel/plugin-proposal-logical-assignment-operators/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-128YbMpjCrP35IOExw2Fq+x55LMP42DzhOhX2aNNIdI9avSWl2PI0yuBWarr3RYpZBSPtabfadkH2yeRiMD61Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-logical-assignment-operators': 7.10.4_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-nullish-coalescing-operator/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-aUOrYU3EVtjf62jQrCj63pYZ7k6vns2h/DQvHPWGmsJRYzWXZ6/AsfgpiRy6XiuIDADhJzP2Q9MwSMKauBQ+UQ==} + /@babel/plugin-proposal-nullish-coalescing-operator/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-numeric-separator/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-vQgPMknOIgiuVqbokToyXbkY/OmmjAzr/0lhSIbG/KmnzXPGwW/AdhdKpi+O4X/VkWiWjnkKOBiqJrTaC98VKw==} + /@babel/plugin-proposal-numeric-separator/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-numeric-separator': 7.10.4_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-numeric-separator': 7.10.4_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-object-rest-spread/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-3O0Y4+dw94HA86qSg9IHfyPktgR7q3gpNVAeiKQd+8jBKFaU5NQS1Yatgo4wY+UFNuLjvxcSmzcsHqrhgTyBUA==} + /@babel/plugin-proposal-object-rest-spread/7.19.4_@babel+core@7.19.6: + resolution: {integrity: sha512-wHmj6LDxVDnL+3WhXteUBaoM1aVILZODAUjg11kHqG4cOlfgMQGxw6aCgvrXrmaJR3Bn14oZhImyCPZzRpC93Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/compat-data': 7.17.7 - '@babel/core': 7.17.9 - '@babel/helper-compilation-targets': 7.17.7_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-object-rest-spread': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-transform-parameters': 7.16.7_@babel+core@7.17.9 + '@babel/compat-data': 7.20.1 + '@babel/core': 7.19.6 + '@babel/helper-compilation-targets': 7.20.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-object-rest-spread': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-transform-parameters': 7.20.1_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-optional-catch-binding/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-eMOH/L4OvWSZAE1VkHbr1vckLG1WUcHGJSLqqQwl2GaUqG6QjddvrOaTUMNYiv77H5IKPMZ9U9P7EaHwvAShfA==} + /@babel/plugin-proposal-optional-catch-binding/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-optional-catch-binding': 7.8.3_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-optional-chaining/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-eC3xy+ZrUcBtP7x+sq62Q/HYd674pPTb/77XZMb5wbDPGWIdUbSr4Agr052+zaUPSb+gGRnjxXfKFvx5iMJ+DA==} + /@babel/plugin-proposal-optional-chaining/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-v5nwt4IqBXihxGsW2QmCWMDS3B3bzGIk/EQVZz2ei7f3NJl8NzAJVvUmpDW5q1CRNY+Beb/k58UAH1Km1N411w==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-skip-transparent-expression-wrappers': 7.16.0 - '@babel/plugin-syntax-optional-chaining': 7.8.3_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 + '@babel/plugin-syntax-optional-chaining': 7.8.3_@babel+core@7.19.6 dev: true - /@babel/plugin-proposal-private-methods/7.16.11_@babel+core@7.17.9: - resolution: {integrity: sha512-F/2uAkPlXDr8+BHpZvo19w3hLFKge+k75XUprE6jaqKxjGkSYcK+4c+bup5PdW/7W/Rpjwql7FTVEDW+fRAQsw==} + /@babel/plugin-proposal-private-methods/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-class-features-plugin': 7.18.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-create-class-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-proposal-private-property-in-object/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-rMQkjcOFbm+ufe3bTZLyOfsOUOxyvLXZJCTARhJr+8UMSoZmqTe1K1BgkFcrW37rAchWg57yI69ORxiWvUINuQ==} + /@babel/plugin-proposal-private-property-in-object/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-9Rysx7FOctvT5ouj5JODjAFAkgGoudQuLPamZb0v1TGLpapdNaftzifU8NTWQm0IRjqoYypdrSmyWgkocDQ8Dw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-create-class-features-plugin': 7.18.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-private-property-in-object': 7.14.5_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-create-class-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-private-property-in-object': 7.14.5_@babel+core@7.19.6 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-proposal-unicode-property-regex/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-QRK0YI/40VLhNVGIjRNAAQkEHws0cswSdFFjpFyt943YmJIU1da9uW63Iu6NFV6CxTZW5eTDCrwZUstBWgp/Rg==} + /@babel/plugin-proposal-unicode-property-regex/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==} engines: {node: '>=4'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-regexp-features-plugin': 7.17.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-create-regexp-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-async-generators/7.8.4_@babel+core@7.17.9: + /@babel/plugin-syntax-async-generators/7.8.4_@babel+core@7.19.6: resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-bigint/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-bigint/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-class-properties/7.12.13_@babel+core@7.17.9: + /@babel/plugin-syntax-class-properties/7.12.13_@babel+core@7.19.6: resolution: {integrity: sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-class-static-block/7.14.5_@babel+core@7.17.9: + /@babel/plugin-syntax-class-static-block/7.14.5_@babel+core@7.19.6: resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-dynamic-import/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-dynamic-import/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-export-default-from/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-4C3E4NsrLOgftKaTYTULhHsuQrGv3FHrBzOMDiS7UYKIpgGBkAdawg4h+EI8zPeK9M0fiIIh72hIwsI24K7MbA==} + /@babel/plugin-syntax-export-default-from/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-Kr//z3ujSVNx6E9z9ih5xXXMqK07VVTuqPmqGe6Mss/zW5XPeLZeSDZoP9ab/hT4wPKqAgjl2PnhPrcpk8Seew==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-export-namespace-from/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-export-namespace-from/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + dev: true + + /@babel/plugin-syntax-import-assertions/7.20.0_@babel+core@7.19.6: + resolution: {integrity: sha512-IUh1vakzNoWalR8ch/areW7qFopR2AEw03JlG7BbrDqmQ4X3q9uuipQwSGrUn7oGiemKjtSLDhNtQHzMHr1JdQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-import-meta/7.10.4_@babel+core@7.17.9: + /@babel/plugin-syntax-import-meta/7.10.4_@babel+core@7.19.6: resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-json-strings/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-json-strings/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-jsx/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-Esxmk7YjA8QysKeT3VhTXvF6y77f/a91SIs4pWb4H2eWGQkCKFgQaG6hdoEVZtGsrAcb2K5BW66XsOErD4WU3Q==} + /@babel/plugin-syntax-jsx/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-logical-assignment-operators/7.10.4_@babel+core@7.17.9: + /@babel/plugin-syntax-logical-assignment-operators/7.10.4_@babel+core@7.19.6: resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-nullish-coalescing-operator/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-nullish-coalescing-operator/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-numeric-separator/7.10.4_@babel+core@7.17.9: + /@babel/plugin-syntax-numeric-separator/7.10.4_@babel+core@7.19.6: resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-object-rest-spread/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-object-rest-spread/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-optional-catch-binding/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-optional-catch-binding/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-optional-chaining/7.8.3_@babel+core@7.17.9: + /@babel/plugin-syntax-optional-chaining/7.8.3_@babel+core@7.19.6: resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-private-property-in-object/7.14.5_@babel+core@7.17.9: + /@babel/plugin-syntax-private-property-in-object/7.14.5_@babel+core@7.19.6: resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-top-level-await/7.14.5_@babel+core@7.17.9: + /@babel/plugin-syntax-top-level-await/7.14.5_@babel+core@7.19.6: resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-syntax-typescript/7.17.12_@babel+core@7.17.9: - resolution: {integrity: sha512-TYY0SXFiO31YXtNg3HtFwNJHjLsAyIIhAhNWkQ5whPPS7HWUFlg9z0Ta4qAQNjQbP1wsSt/oKkmZ/4/WWdMUpw==} + /@babel/plugin-syntax-typescript/7.20.0_@babel+core@7.19.6: + resolution: {integrity: sha512-rd9TkG+u1CExzS4SM1BlMEhMXwFLKVjOAFFCDx9PbX5ycJWDoWMcwdJH9RhkPu1dOgn5TrxLot/Gx6lWFuAUNQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + dev: true - /@babel/plugin-transform-arrow-functions/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-9ffkFFMbvzTvv+7dTp/66xvZAWASuPD5Tl9LK3Z9vhOmANo6j94rik+5YMBt4CwHVMWLWpMsriIc2zsa3WW3xQ==} + /@babel/plugin-transform-arrow-functions/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-async-to-generator/7.16.8_@babel+core@7.17.9: - resolution: {integrity: sha512-MtmUmTJQHCnyJVrScNzNlofQJ3dLFuobYn3mwOTKHnSCMtbNsqvF71GQmJfFjdrXSsAA7iysFmYWw4bXZ20hOg==} + /@babel/plugin-transform-async-to-generator/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-module-imports': 7.16.7 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-remap-async-to-generator': 7.16.8 + '@babel/core': 7.19.6 + '@babel/helper-module-imports': 7.18.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-remap-async-to-generator': 7.18.9_@babel+core@7.19.6 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-block-scoped-functions/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-JUuzlzmF40Z9cXyytcbZEZKckgrQzChbQJw/5PuEHYeqzCsvebDx0K0jWnIIVcmmDOAVctCgnYs0pMcrYj2zJg==} + /@babel/plugin-transform-block-scoped-functions/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-block-scoping/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-ObZev2nxVAYA4bhyusELdo9hb3H+A56bxH3FZMbEImZFiEDYVHXQSJ1hQKFlDnlt8G9bBrCZ5ZpURZUrV4G5qQ==} + /@babel/plugin-transform-block-scoping/7.20.0_@babel+core@7.19.6: + resolution: {integrity: sha512-sXOohbpHZSk7GjxK9b3dKB7CfqUD5DwOH+DggKzOQ7TXYP+RCSbRykfjQmn/zq+rBjycVRtLf9pYhAaEJA786w==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-classes/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-WY7og38SFAGYRe64BrjKf8OrE6ulEHtr5jEYaZMwox9KebgqPi67Zqz8K53EKk1fFEJgm96r32rkKZ3qA2nCWQ==} + /@babel/plugin-transform-classes/7.19.0_@babel+core@7.19.6: + resolution: {integrity: sha512-YfeEE9kCjqTS9IitkgfJuxjcEtLUHMqa8yUJ6zdz8vR7hKuo6mOy2C05P0F1tdMmDCeuyidKnlrw/iTppHcr2A==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-environment-visitor': 7.16.7 - '@babel/helper-function-name': 7.17.9 - '@babel/helper-optimise-call-expression': 7.16.7 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-replace-supers': 7.16.7 - '@babel/helper-split-export-declaration': 7.16.7 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-compilation-targets': 7.20.0_@babel+core@7.19.6 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-function-name': 7.19.0 + '@babel/helper-optimise-call-expression': 7.18.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-replace-supers': 7.19.1 + '@babel/helper-split-export-declaration': 7.18.6 globals: 11.12.0 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-computed-properties/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-gN72G9bcmenVILj//sv1zLNaPyYcOzUho2lIJBMh/iakJ9ygCo/hEF9cpGb61SCMEDxbbyBoVQxrt+bWKu5KGw==} + /@babel/plugin-transform-computed-properties/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-+i0ZU1bCDymKakLxn5srGHrsAPRELC2WIbzwjLhHW9SIE1cPYkLCL0NlnXMZaM1vhfgA2+M7hySk42VBvrkBRw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-destructuring/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-VqAwhTHBnu5xBVDCvrvqJbtLUa++qZaWC0Fgr2mqokBlulZARGyIvZDoqbPlPaKImQ9dKAcCzbv+ul//uqu70A==} + /@babel/plugin-transform-destructuring/7.20.0_@babel+core@7.19.6: + resolution: {integrity: sha512-1dIhvZfkDVx/zn2S1aFwlruspTt4189j7fEkH0Y0VyuDM6bQt7bD6kLcz3l4IlLG+e5OReaBz9ROAbttRtUHqA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-dotall-regex/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-Lyttaao2SjZF6Pf4vk1dVKv8YypMpomAbygW+mU5cYP3S5cWTfCJjG8xV6CFdzGFlfWK81IjL9viiTvpb6G7gQ==} + /@babel/plugin-transform-dotall-regex/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-regexp-features-plugin': 7.17.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-create-regexp-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-duplicate-keys/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-03DvpbRfvWIXyK0/6QiR1KMTWeT6OcQ7tbhjrXyFS02kjuX/mu5Bvnh5SDSWHxyawit2g5aWhKwI86EE7GUnTw==} + /@babel/plugin-transform-duplicate-keys/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-d2bmXCtZXYc59/0SanQKbiWINadaJXqtvIQIzd4+hNwkWBgyCd5F/2t1kXoUdvPMrxzPvhK6EMQRROxsue+mfw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-exponentiation-operator/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-8UYLSlyLgRixQvlYH3J2ekXFHDFLQutdy7FfFAMm3CPZ6q9wHCwnUyiXpQCe3gVVnQlHc5nsuiEVziteRNTXEA==} + /@babel/plugin-transform-exponentiation-operator/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-builder-binary-assignment-operator-visitor': 7.16.7 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-builder-binary-assignment-operator-visitor': 7.18.9 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-for-of/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-/QZm9W92Ptpw7sjI9Nx1mbcsWz33+l8kuMIQnDwgQBG5s3fAfQvkRjQ7NqXhtNcKOnPkdICmUHyCaWW06HCsqg==} + /@babel/plugin-transform-for-of/7.18.8_@babel+core@7.19.6: + resolution: {integrity: sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-function-name/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-SU/C68YVwTRxqWj5kgsbKINakGag0KTgq9f2iZEXdStoAbOzLHEBRYzImmA6yFo8YZhJVflvXmIHUO7GWHmxxA==} + /@babel/plugin-transform-function-name/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-WvIBoRPaJQ5yVHzcnJFor7oS5Ls0PYixlTYE63lCj2RtdQEl15M68FXQlxnG6wdraJIXRdR7KI+hQ7q/9QjrCQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-compilation-targets': 7.17.7_@babel+core@7.17.9 - '@babel/helper-function-name': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-compilation-targets': 7.20.0_@babel+core@7.19.6 + '@babel/helper-function-name': 7.19.0 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-literals/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-6tH8RTpTWI0s2sV6uq3e/C9wPo4PTqqZps4uF0kzQ9/xPLFQtipynvmT1g/dOfEJ+0EQsHhkQ/zyRId8J2b8zQ==} + /@babel/plugin-transform-literals/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-member-expression-literals/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-mBruRMbktKQwbxaJof32LT9KLy2f3gH+27a5XSuXo6h7R3vqltl0PgZ80C8ZMKw98Bf8bqt6BEVi3svOh2PzMw==} + /@babel/plugin-transform-member-expression-literals/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-modules-amd/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-KaaEtgBL7FKYwjJ/teH63oAmE3lP34N3kshz8mm4VMAw7U3PxjVwwUmxEFksbgsNUaO3wId9R2AVQYSEGRa2+g==} + /@babel/plugin-transform-modules-amd/7.19.6_@babel+core@7.19.6: + resolution: {integrity: sha512-uG3od2mXvAtIFQIh0xrpLH6r5fpSQN04gIVovl+ODLdUMANokxQLZnPBHcjmv3GxRjnqwLuHvppjjcelqUFZvg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-module-transforms': 7.17.7 - '@babel/helper-plugin-utils': 7.17.12 - babel-plugin-dynamic-import-node: 2.3.3 + '@babel/core': 7.19.6 + '@babel/helper-module-transforms': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-modules-commonjs/7.16.8_@babel+core@7.17.9: - resolution: {integrity: sha512-oflKPvsLT2+uKQopesJt3ApiaIS2HW+hzHFcwRNtyDGieAeC/dIHZX8buJQ2J2X1rxGPy4eRcUijm3qcSPjYcA==} + /@babel/plugin-transform-modules-commonjs/7.19.6_@babel+core@7.19.6: + resolution: {integrity: sha512-8PIa1ym4XRTKuSsOUXqDG0YaOlEuTVvHMe5JCfgBMOtHvJKw/4NGovEGN33viISshG/rZNVrACiBmPQLvWN8xQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-module-transforms': 7.17.7 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-simple-access': 7.17.7 - babel-plugin-dynamic-import-node: 2.3.3 + '@babel/core': 7.19.6 + '@babel/helper-module-transforms': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-simple-access': 7.19.4 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-modules-systemjs/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-DuK5E3k+QQmnOqBR9UkusByy5WZWGRxfzV529s9nPra1GE7olmxfqO2FHobEOYSPIjPBTr4p66YDcjQnt8cBmw==} + /@babel/plugin-transform-modules-systemjs/7.19.6_@babel+core@7.19.6: + resolution: {integrity: sha512-fqGLBepcc3kErfR9R3DnVpURmckXP7gj7bAlrTQyBxrigFqszZCkFkcoxzCp2v32XmwXLvbw+8Yq9/b+QqksjQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-hoist-variables': 7.16.7 - '@babel/helper-module-transforms': 7.17.7 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-validator-identifier': 7.16.7 - babel-plugin-dynamic-import-node: 2.3.3 + '@babel/core': 7.19.6 + '@babel/helper-hoist-variables': 7.18.6 + '@babel/helper-module-transforms': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-validator-identifier': 7.19.1 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-modules-umd/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-EMh7uolsC8O4xhudF2F6wedbSHm1HHZ0C6aJ7K67zcDNidMzVcxWdGr+htW9n21klm+bOn+Rx4CBsAntZd3rEQ==} + /@babel/plugin-transform-modules-umd/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-module-transforms': 7.17.7 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-module-transforms': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-named-capturing-groups-regex/7.16.8_@babel+core@7.17.9: - resolution: {integrity: sha512-j3Jw+n5PvpmhRR+mrgIh04puSANCk/T/UA3m3P1MjJkhlK906+ApHhDIqBQDdOgL/r1UYpz4GNclTXxyZrYGSw==} + /@babel/plugin-transform-named-capturing-groups-regex/7.19.1_@babel+core@7.19.6: + resolution: {integrity: sha512-oWk9l9WItWBQYS4FgXD4Uyy5kq898lvkXpXQxoJEY1RnvPk4R/Dvu2ebXU9q8lP+rlMwUQTFf2Ok6d78ODa0kw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-regexp-features-plugin': 7.17.0_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-create-regexp-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-new-target/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-xiLDzWNMfKoGOpc6t3U+etCE2yRnn3SM09BXqWPIZOBpL2gvVrBWUKnsJx0K/ADi5F5YC5f8APFfWrz25TdlGg==} + /@babel/plugin-transform-new-target/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-object-super/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-14J1feiQVWaGvRxj2WjyMuXS2jsBkgB3MdSN5HuC2G5nRspa5RK9COcs82Pwy5BuGcjb+fYaUj94mYcOj7rCvw==} + /@babel/plugin-transform-object-super/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-replace-supers': 7.16.7 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-replace-supers': 7.19.1 transitivePeerDependencies: - supports-color dev: true - /@babel/plugin-transform-parameters/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-AT3MufQ7zZEhU2hwOA11axBnExW0Lszu4RL/tAlUJBuNoRak+wehQW8h6KcXOcgjY42fHtDxswuMhMjFEuv/aw==} + /@babel/plugin-transform-parameters/7.20.1_@babel+core@7.19.6: + resolution: {integrity: sha512-nDvKLrAvl+kf6BOy1UJ3MGwzzfTMgppxwiD2Jb4LO3xjYyZq30oQzDNJbCQpMdG9+j2IXHoiMrw5Cm/L6ZoxXQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-property-literals/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-z4FGr9NMGdoIl1RqavCqGG+ZuYjfZ/hkCIeuH6Do7tXmSm0ls11nYVSJqFEUOSJbDab5wC6lRE/w6YjVcr6Hqw==} + /@babel/plugin-transform-property-literals/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-react-display-name/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-qgIg8BcZgd0G/Cz916D5+9kqX0c7nPZyXaP8R2tLNN5tkyIZdG5fEwBrxwplzSnjC1jvQmyMNVwUCZPcbGY7Pg==} + /@babel/plugin-transform-react-display-name/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-TV4sQ+T013n61uMoygyMRm+xf04Bd5oqFpv2jAEQwSZ8NwQA7zeRPg1LMVg2PWi3zWBz+CLKD+v5bcpZ/BS0aA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-react-jsx-development/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-RMvQWvpla+xy6MlBpPlrKZCMRs2AGiHOGHY3xRwl0pEeim348dDyxeH4xBsMPbIMhujeq7ihE702eM2Ew0Wo+A==} + /@babel/plugin-transform-react-jsx-development/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-SA6HEjwYFKF7WDjWcMcMGUimmw/nhNRDWxr+KaLSCrkD/LMDBvWRmHAYgE1HDeF8KUuI8OAu+RT6EOtKxSW2qA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/plugin-transform-react-jsx': 7.16.7_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/plugin-transform-react-jsx': 7.19.0_@babel+core@7.19.6 dev: true - /@babel/plugin-transform-react-jsx/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-8D16ye66fxiE8m890w0BpPpngG9o9OVBBy0gH2E+2AR7qMR2ZpTYJEqLxAsoroenMId0p/wMW+Blc0meDgu0Ag==} + /@babel/plugin-transform-react-jsx/7.19.0_@babel+core@7.19.6: + resolution: {integrity: sha512-UVEvX3tXie3Szm3emi1+G63jyw1w5IcMY0FSKM+CRnKRI5Mr1YbCNgsSTwoTwKphQEG9P+QqmuRFneJPZuHNhg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-module-imports': 7.16.7 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-jsx': 7.16.7_@babel+core@7.17.9 - '@babel/types': 7.17.0 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-module-imports': 7.18.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-jsx': 7.18.6_@babel+core@7.19.6 + '@babel/types': 7.20.0 dev: true - /@babel/plugin-transform-react-pure-annotations/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-hs71ToC97k3QWxswh2ElzMFABXHvGiJ01IB1TbYQDGeWRKWz/MPUTh5jGExdHvosYKpnJW5Pm3S4+TA3FyX+GA==} + /@babel/plugin-transform-react-pure-annotations/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-I8VfEPg9r2TRDdvnHgPepTKvuRomzA8+u+nhY7qSI1fR2hRNebasZEETLyM5mAUr0Ku56OkXJ0I7NHJnO6cJiQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-regenerator/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-mF7jOgGYCkSJagJ6XCujSQg+6xC1M77/03K2oBmVJWoFGNUtnVJO4WHKJk3dnPC8HCcj4xBQP1Egm8DWh3Pb3Q==} + /@babel/plugin-transform-regenerator/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-poqRI2+qiSdeldcz4wTSTXBRryoq3Gc70ye7m7UD5Ww0nE29IXqMl6r7Nd15WBgRd74vloEMlShtH6CKxVzfmQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - regenerator-transform: 0.14.5 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + regenerator-transform: 0.15.0 dev: true - /@babel/plugin-transform-reserved-words/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-KQzzDnZ9hWQBjwi5lpY5v9shmm6IVG0U9pB18zvMu2i4H90xpT4gmqwPYsn8rObiadYe2M0gmgsiOIF5A/2rtg==} + /@babel/plugin-transform-reserved-words/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-shorthand-properties/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-hah2+FEnoRoATdIb05IOXf+4GzXYTq75TVhIn1PewihbpyrNWUt2JbudKQOETWw6QpLe+AIUpJ5MVLYTQbeeUg==} + /@babel/plugin-transform-shorthand-properties/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-spread/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-+pjJpgAngb53L0iaA5gU/1MLXJIfXcYepLgXB3esVRf4fqmj8f2cxM3/FKaHsZms08hFQJkFccEWuIpm429TXg==} + /@babel/plugin-transform-spread/7.19.0_@babel+core@7.19.6: + resolution: {integrity: sha512-RsuMk7j6n+r752EtzyScnWkQyuJdli6LdO5Klv8Yx0OfPVTcQkIUfS8clx5e9yHXzlnhOZF3CbQ8C2uP5j074w==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-skip-transparent-expression-wrappers': 7.16.0 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 dev: true - /@babel/plugin-transform-sticky-regex/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-NJa0Bd/87QV5NZZzTuZG5BPJjLYadeSZ9fO6oOUoL4iQx+9EEuw/eEM92SrsT19Yc2jgB1u1hsjqDtH02c3Drw==} + /@babel/plugin-transform-sticky-regex/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-template-literals/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-VwbkDDUeenlIjmfNeDX/V0aWrQH2QiVyJtwymVQSzItFDTpxfyJh3EVaQiS0rIN/CqbLGr0VcGmuwyTdZtdIsA==} + /@babel/plugin-transform-template-literals/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-typeof-symbol/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-p2rOixCKRJzpg9JB4gjnG4gjWkWa89ZoYUnl9snJ1cWIcTH/hvxZqfO+WjG6T8DRBpctEol5jw1O5rA8gkCokQ==} + /@babel/plugin-transform-typeof-symbol/7.18.9_@babel+core@7.19.6: + resolution: {integrity: sha512-SRfwTtF11G2aemAZWivL7PD+C9z52v9EvMqH9BuYbabyPuKUvSWks3oCg6041pT925L4zVFqaVBeECwsmlguEw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-typescript/7.18.4_@babel+core@7.17.9: - resolution: {integrity: sha512-l4vHuSLUajptpHNEOUDEGsnpl9pfRLsN1XUoDQDD/YBuXTM+v37SHGS+c6n4jdcZy96QtuUuSvZYMLSSsjH8Mw==} + /@babel/plugin-transform-typescript/7.20.0_@babel+core@7.19.6: + resolution: {integrity: sha512-xOAsAFaun3t9hCwZ13Qe7gq423UgMZ6zAgmLxeGGapFqlT/X3L5qT2btjiVLlFn7gWtMaVyceS5VxGAuKbgizw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-class-features-plugin': 7.18.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-syntax-typescript': 7.17.12_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-create-class-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-syntax-typescript': 7.20.0_@babel+core@7.19.6 transitivePeerDependencies: - supports-color - dev: false + dev: true - /@babel/plugin-transform-unicode-escapes/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-TAV5IGahIz3yZ9/Hfv35TV2xEm+kaBDaZQCn2S/hG9/CZ0DktxJv9eKfPc7yYCvOYR4JGx1h8C+jcSOvgaaI/Q==} + /@babel/plugin-transform-unicode-escapes/7.18.10_@babel+core@7.19.6: + resolution: {integrity: sha512-kKAdAI+YzPgGY/ftStBFXTI1LZFju38rYThnfMykS+IXy8BVx+res7s2fxf1l8I35DV2T97ezo6+SGrXz6B3iQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/plugin-transform-unicode-regex/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-oC5tYYKw56HO75KZVLQ+R/Nl3Hro9kf8iG0hXoaHP7tjAyCpvqBiSNe6vGrZni1Z6MggmUOC6A7VP7AVmw225Q==} + /@babel/plugin-transform-unicode-regex/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-create-regexp-features-plugin': 7.17.0_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 + '@babel/core': 7.19.6 + '@babel/helper-create-regexp-features-plugin': 7.19.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 dev: true - /@babel/preset-env/7.16.11_@babel+core@7.17.9: - resolution: {integrity: sha512-qcmWG8R7ZW6WBRPZK//y+E3Cli151B20W1Rv7ln27vuPaXU/8TKms6jFdiJtF7UDTxcrb7mZd88tAeK9LjdT8g==} + /@babel/preset-env/7.19.4_@babel+core@7.19.6: + resolution: {integrity: sha512-5QVOTXUdqTCjQuh2GGtdd7YEhoRXBMVGROAtsBeLGIbIz3obCBIfRMT1I3ZKkMgNzwkyCkftDXSSkHxnfVf4qg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/compat-data': 7.17.7 - '@babel/core': 7.17.9 - '@babel/helper-compilation-targets': 7.17.7_@babel+core@7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-validator-option': 7.16.7 - '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-async-generator-functions': 7.16.8_@babel+core@7.17.9 - '@babel/plugin-proposal-class-properties': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-class-static-block': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-dynamic-import': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-export-namespace-from': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-json-strings': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-logical-assignment-operators': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-nullish-coalescing-operator': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-numeric-separator': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-object-rest-spread': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-optional-catch-binding': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-optional-chaining': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-private-methods': 7.16.11_@babel+core@7.17.9 - '@babel/plugin-proposal-private-property-in-object': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-proposal-unicode-property-regex': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-syntax-async-generators': 7.8.4_@babel+core@7.17.9 - '@babel/plugin-syntax-class-properties': 7.12.13_@babel+core@7.17.9 - '@babel/plugin-syntax-class-static-block': 7.14.5_@babel+core@7.17.9 - '@babel/plugin-syntax-dynamic-import': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-export-namespace-from': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-json-strings': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-logical-assignment-operators': 7.10.4_@babel+core@7.17.9 - '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-numeric-separator': 7.10.4_@babel+core@7.17.9 - '@babel/plugin-syntax-object-rest-spread': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-optional-catch-binding': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-optional-chaining': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-private-property-in-object': 7.14.5_@babel+core@7.17.9 - '@babel/plugin-syntax-top-level-await': 7.14.5_@babel+core@7.17.9 - '@babel/plugin-transform-arrow-functions': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-async-to-generator': 7.16.8_@babel+core@7.17.9 - '@babel/plugin-transform-block-scoped-functions': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-block-scoping': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-classes': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-computed-properties': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-destructuring': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-dotall-regex': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-duplicate-keys': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-exponentiation-operator': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-for-of': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-function-name': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-literals': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-member-expression-literals': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-modules-amd': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-modules-commonjs': 7.16.8_@babel+core@7.17.9 - '@babel/plugin-transform-modules-systemjs': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-modules-umd': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-named-capturing-groups-regex': 7.16.8_@babel+core@7.17.9 - '@babel/plugin-transform-new-target': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-object-super': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-parameters': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-property-literals': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-regenerator': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-reserved-words': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-shorthand-properties': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-spread': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-sticky-regex': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-template-literals': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-typeof-symbol': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-unicode-escapes': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-unicode-regex': 7.16.7_@babel+core@7.17.9 - '@babel/preset-modules': 0.1.5_@babel+core@7.17.9 - '@babel/types': 7.17.0 - babel-plugin-polyfill-corejs2: 0.3.1_@babel+core@7.17.9 - babel-plugin-polyfill-corejs3: 0.5.2_@babel+core@7.17.9 - babel-plugin-polyfill-regenerator: 0.3.1_@babel+core@7.17.9 - core-js-compat: 3.21.0 + '@babel/compat-data': 7.20.1 + '@babel/core': 7.19.6 + '@babel/helper-compilation-targets': 7.20.0_@babel+core@7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-validator-option': 7.18.6 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-proposal-async-generator-functions': 7.20.1_@babel+core@7.19.6 + '@babel/plugin-proposal-class-properties': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-class-static-block': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-dynamic-import': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-export-namespace-from': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-proposal-json-strings': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-logical-assignment-operators': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-numeric-separator': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-object-rest-spread': 7.19.4_@babel+core@7.19.6 + '@babel/plugin-proposal-optional-catch-binding': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-optional-chaining': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-proposal-private-methods': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-private-property-in-object': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-proposal-unicode-property-regex': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-syntax-async-generators': 7.8.4_@babel+core@7.19.6 + '@babel/plugin-syntax-class-properties': 7.12.13_@babel+core@7.19.6 + '@babel/plugin-syntax-class-static-block': 7.14.5_@babel+core@7.19.6 + '@babel/plugin-syntax-dynamic-import': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-export-namespace-from': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-import-assertions': 7.20.0_@babel+core@7.19.6 + '@babel/plugin-syntax-json-strings': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4_@babel+core@7.19.6 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-numeric-separator': 7.10.4_@babel+core@7.19.6 + '@babel/plugin-syntax-object-rest-spread': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-optional-chaining': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-private-property-in-object': 7.14.5_@babel+core@7.19.6 + '@babel/plugin-syntax-top-level-await': 7.14.5_@babel+core@7.19.6 + '@babel/plugin-transform-arrow-functions': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-async-to-generator': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-block-scoped-functions': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-block-scoping': 7.20.0_@babel+core@7.19.6 + '@babel/plugin-transform-classes': 7.19.0_@babel+core@7.19.6 + '@babel/plugin-transform-computed-properties': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-transform-destructuring': 7.20.0_@babel+core@7.19.6 + '@babel/plugin-transform-dotall-regex': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-duplicate-keys': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-transform-exponentiation-operator': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-for-of': 7.18.8_@babel+core@7.19.6 + '@babel/plugin-transform-function-name': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-transform-literals': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-transform-member-expression-literals': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-modules-amd': 7.19.6_@babel+core@7.19.6 + '@babel/plugin-transform-modules-commonjs': 7.19.6_@babel+core@7.19.6 + '@babel/plugin-transform-modules-systemjs': 7.19.6_@babel+core@7.19.6 + '@babel/plugin-transform-modules-umd': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-named-capturing-groups-regex': 7.19.1_@babel+core@7.19.6 + '@babel/plugin-transform-new-target': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-object-super': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-parameters': 7.20.1_@babel+core@7.19.6 + '@babel/plugin-transform-property-literals': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-regenerator': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-reserved-words': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-shorthand-properties': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-spread': 7.19.0_@babel+core@7.19.6 + '@babel/plugin-transform-sticky-regex': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-template-literals': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-transform-typeof-symbol': 7.18.9_@babel+core@7.19.6 + '@babel/plugin-transform-unicode-escapes': 7.18.10_@babel+core@7.19.6 + '@babel/plugin-transform-unicode-regex': 7.18.6_@babel+core@7.19.6 + '@babel/preset-modules': 0.1.5_@babel+core@7.19.6 + '@babel/types': 7.20.0 + babel-plugin-polyfill-corejs2: 0.3.3_@babel+core@7.19.6 + babel-plugin-polyfill-corejs3: 0.6.0_@babel+core@7.19.6 + babel-plugin-polyfill-regenerator: 0.4.1_@babel+core@7.19.6 + core-js-compat: 3.26.0 semver: 6.3.0 transitivePeerDependencies: - supports-color dev: true - /@babel/preset-modules/0.1.5_@babel+core@7.17.9: + /@babel/preset-modules/0.1.5_@babel+core@7.19.6: resolution: {integrity: sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/plugin-proposal-unicode-property-regex': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-dotall-regex': 7.16.7_@babel+core@7.17.9 - '@babel/types': 7.17.0 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/plugin-proposal-unicode-property-regex': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-dotall-regex': 7.18.6_@babel+core@7.19.6 + '@babel/types': 7.20.0 esutils: 2.0.3 dev: true - /@babel/preset-react/7.16.7_@babel+core@7.17.9: - resolution: {integrity: sha512-fWpyI8UM/HE6DfPBzD8LnhQ/OcH8AgTaqcqP2nGOXEUV+VKBR5JRN9hCk9ai+zQQ57vtm9oWeXguBCPNUjytgA==} + /@babel/preset-react/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-zXr6atUmyYdiWRVLOZahakYmOBHtWc2WGCkP8PYTgZi0iJXDY2CN180TdrIW4OGOAdLc7TifzDIvtx6izaRIzg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-validator-option': 7.16.7 - '@babel/plugin-transform-react-display-name': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-react-jsx': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-react-jsx-development': 7.16.7_@babel+core@7.17.9 - '@babel/plugin-transform-react-pure-annotations': 7.16.7_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-validator-option': 7.18.6 + '@babel/plugin-transform-react-display-name': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-react-jsx': 7.19.0_@babel+core@7.19.6 + '@babel/plugin-transform-react-jsx-development': 7.18.6_@babel+core@7.19.6 + '@babel/plugin-transform-react-pure-annotations': 7.18.6_@babel+core@7.19.6 dev: true - /@babel/preset-typescript/7.17.12_@babel+core@7.17.9: - resolution: {integrity: sha512-S1ViF8W2QwAKUGJXxP9NAfNaqGDdEBJKpYkxHf5Yy2C4NPPzXGeR3Lhk7G8xJaaLcFTRfNjVbtbVtm8Gb0mqvg==} + /@babel/preset-typescript/7.18.6_@babel+core@7.19.6: + resolution: {integrity: sha512-s9ik86kXBAnD760aybBucdpnLsAt0jK1xqJn2juOn9lkOvSHV60os5hxoVJsPzMQxvnUJFAlkont2DvvaYEBtQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-plugin-utils': 7.17.12 - '@babel/helper-validator-option': 7.16.7 - '@babel/plugin-transform-typescript': 7.18.4_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-plugin-utils': 7.19.0 + '@babel/helper-validator-option': 7.18.6 + '@babel/plugin-transform-typescript': 7.20.0_@babel+core@7.19.6 transitivePeerDependencies: - supports-color - dev: false + dev: true /@babel/runtime-corejs3/7.13.10: resolution: {integrity: sha512-x/XYVQ1h684pp1mJwOV4CyvqZXqbc8CMsMGUnAbuc82ZCdv1U63w5RSUzgDSXQHG5Rps/kiksH6g2D5BuaKyXg==} dependencies: core-js-pure: 3.9.1 - regenerator-runtime: 0.13.8 + regenerator-runtime: 0.13.10 dev: true - /@babel/runtime/7.17.9: - resolution: {integrity: sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==} + /@babel/runtime/7.20.1: + resolution: {integrity: sha512-mrzLkl6U9YLF8qpqI7TB82PESyEGjm/0Ly91jG575eVxMMlb8fYfOXFZIJ8XfLrJZQbm7dlKry2bJmXBUEkdFg==} engines: {node: '>=6.9.0'} dependencies: - regenerator-runtime: 0.13.8 + regenerator-runtime: 0.13.10 - /@babel/template/7.16.7: - resolution: {integrity: sha512-I8j/x8kHUrbYRTUxXrrMbfCa7jxkE7tZre39x3kjr9hvI82cK1FfqLygotcWN5kdPGWcLdWMHpSBavse5tWw3w==} + /@babel/template/7.18.10: + resolution: {integrity: sha512-TI+rCtooWHr3QJ27kJxfjutghu44DLnasDMwpDqCXVTal9RLp3RSYNh4NdBrRP2cQAoG9A8juOQl6P6oZG4JxA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/code-frame': 7.16.7 - '@babel/parser': 7.17.9 - '@babel/types': 7.17.0 + '@babel/code-frame': 7.18.6 + '@babel/parser': 7.20.1 + '@babel/types': 7.20.0 - /@babel/traverse/7.17.9: + /@babel/traverse/7.17.9_supports-color@5.5.0: resolution: {integrity: sha512-PQO8sDIJ8SIwipTPiR71kJQCKQYB5NGImbOviK8K+kg5xkNSYXLBupuX9QhatFowrsvo9Hj8WgArg3W7ijNAQw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/code-frame': 7.16.7 - '@babel/generator': 7.17.9 - '@babel/helper-environment-visitor': 7.16.7 - '@babel/helper-function-name': 7.17.9 - '@babel/helper-hoist-variables': 7.16.7 - '@babel/helper-split-export-declaration': 7.16.7 - '@babel/parser': 7.17.9 - '@babel/types': 7.17.0 - debug: 4.3.4 + '@babel/code-frame': 7.18.6 + '@babel/generator': 7.20.1 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-function-name': 7.19.0 + '@babel/helper-hoist-variables': 7.18.6 + '@babel/helper-split-export-declaration': 7.18.6 + '@babel/parser': 7.20.1 + '@babel/types': 7.20.0 + debug: 4.3.4_supports-color@5.5.0 globals: 11.12.0 transitivePeerDependencies: - supports-color + dev: false - /@babel/traverse/7.17.9_supports-color@5.5.0: - resolution: {integrity: sha512-PQO8sDIJ8SIwipTPiR71kJQCKQYB5NGImbOviK8K+kg5xkNSYXLBupuX9QhatFowrsvo9Hj8WgArg3W7ijNAQw==} + /@babel/traverse/7.20.1: + resolution: {integrity: sha512-d3tN8fkVJwFLkHkBN479SOsw4DMZnz8cdbL/gvuDuzy3TS6Nfw80HuQqhw1pITbIruHyh7d1fMA47kWzmcUEGA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/code-frame': 7.16.7 - '@babel/generator': 7.17.9 - '@babel/helper-environment-visitor': 7.16.7 - '@babel/helper-function-name': 7.17.9 - '@babel/helper-hoist-variables': 7.16.7 - '@babel/helper-split-export-declaration': 7.16.7 - '@babel/parser': 7.17.9 - '@babel/types': 7.17.0 - debug: 4.3.4_supports-color@5.5.0 + '@babel/code-frame': 7.18.6 + '@babel/generator': 7.20.1 + '@babel/helper-environment-visitor': 7.18.9 + '@babel/helper-function-name': 7.19.0 + '@babel/helper-hoist-variables': 7.18.6 + '@babel/helper-split-export-declaration': 7.18.6 + '@babel/parser': 7.20.1 + '@babel/types': 7.20.0 + debug: 4.3.4 globals: 11.12.0 transitivePeerDependencies: - supports-color - dev: false + dev: true - /@babel/types/7.17.0: - resolution: {integrity: sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==} + /@babel/types/7.20.0: + resolution: {integrity: sha512-Jlgt3H0TajCW164wkTOTzHkZb075tMQMULzrLUoUeKmO7eFL96GgDxf7/Axhc5CAuKE3KFyVW1p6ysKsi2oXAg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-validator-identifier': 7.16.7 + '@babel/helper-string-parser': 7.19.4 + '@babel/helper-validator-identifier': 7.19.1 to-fast-properties: 2.0.0 /@bcoe/v8-coverage/0.2.3: @@ -2250,7 +2286,7 @@ packages: resolution: {integrity: sha512-ipON6WtYgl/1329g5AIJVbUuEh0wZVbdpGwC99Jw4LwuoBNS95MVphU6zOeD9pDkon+LLbFL7lOQRapbB8SCHw==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@babel/core': 7.17.9 + '@babel/core': 7.19.6 '@jest/types': 27.5.1 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 @@ -2280,18 +2316,45 @@ packages: chalk: 4.1.2 dev: true + /@jridgewell/gen-mapping/0.3.2: + resolution: {integrity: sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/set-array': 1.1.2 + '@jridgewell/sourcemap-codec': 1.4.10 + '@jridgewell/trace-mapping': 0.3.17 + /@jridgewell/resolve-uri/3.0.4: resolution: {integrity: sha512-cz8HFjOFfUBtvN+NXYSFMHYRdxZMaEl0XypVrhzxBgadKIXhIkRd8aMeHhmF56Sl7SuS8OnUpQ73/k9LE4VnLg==} engines: {node: '>=6.0.0'} + dev: true + + /@jridgewell/resolve-uri/3.1.0: + resolution: {integrity: sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==} + engines: {node: '>=6.0.0'} + + /@jridgewell/set-array/1.1.2: + resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} + engines: {node: '>=6.0.0'} /@jridgewell/sourcemap-codec/1.4.10: resolution: {integrity: sha512-Ht8wIW5v165atIX1p+JvKR5ONzUyF4Ac8DZIQ5kZs9zrb6M8SJNXpx1zn04rn65VjBMygRoMXcyYwNK0fT7bEg==} + /@jridgewell/sourcemap-codec/1.4.14: + resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==} + + /@jridgewell/trace-mapping/0.3.17: + resolution: {integrity: sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==} + dependencies: + '@jridgewell/resolve-uri': 3.1.0 + '@jridgewell/sourcemap-codec': 1.4.14 + /@jridgewell/trace-mapping/0.3.2: resolution: {integrity: sha512-9KzzH4kMjA2XmBRHfqG2/Vtl7s92l6uNDd0wW7frDE+EUvQFGqNXhWp0UGJjSkt3v2AYjzOZn1QO9XaTNJIt1Q==} dependencies: '@jridgewell/resolve-uri': 3.0.4 '@jridgewell/sourcemap-codec': 1.4.10 + dev: true /@loaders.gl/3d-tiles/3.2.4_@loaders.gl+core@3.2.4: resolution: {integrity: sha512-qEY1QgBIiRHZYBYWbSGHcSuyKLXbHDS08IYgNkKmGdrdX+rgt73ucgQpIyhjQCA7lxJUF5xewYXEYljvPRVOdw==} @@ -2311,7 +2374,7 @@ packages: /@loaders.gl/core/3.2.4: resolution: {integrity: sha512-SFBw6p8WXUGn8fgpMaGVqTq2nabu5ILJKzr0Q0muN8ESnOpzsdRzI6aXw8Ip8YCerqjklXnuB0Lqeu34odalNw==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@loaders.gl/loader-utils': 3.2.4 '@loaders.gl/worker-utils': 3.2.4 '@probe.gl/log': 3.5.0 @@ -2321,7 +2384,7 @@ packages: /@loaders.gl/draco/3.2.4: resolution: {integrity: sha512-cagaHY5I1TSBPOUyQS0ZWgLtMqd+Dnx9wUzduAxLXEVBNHIS2kuG/FXJa/vZFHI79++ZG48BnbmDo05ZVUd4GA==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@loaders.gl/loader-utils': 3.2.4 '@loaders.gl/schema': 3.2.4 '@loaders.gl/worker-utils': 3.2.4 @@ -2356,7 +2419,7 @@ packages: /@loaders.gl/loader-utils/3.2.4: resolution: {integrity: sha512-FwWgQztK5l0Gn7UaIq1LzSKo4751XY1SlQGxibczxRbkwfpbgyCn9yAyJy59HxcqZboYzXi+mrVr7yxLTYAIQg==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@loaders.gl/worker-utils': 3.2.4 '@probe.gl/stats': 3.5.0 dev: false @@ -2389,7 +2452,7 @@ packages: /@loaders.gl/terrain/3.2.4: resolution: {integrity: sha512-+41KWTn/wm62U+7Nx73tHn1PU8XHUIFK700Diq+VJaien+lbdhIRcqwgQV4NXRjxI/7p1yKn9+YNMGtN08diyA==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@loaders.gl/loader-utils': 3.2.4 '@loaders.gl/schema': 3.2.4 '@mapbox/martini': 0.2.0 @@ -2424,7 +2487,7 @@ packages: /@loaders.gl/worker-utils/3.2.4: resolution: {integrity: sha512-gGy8+LiyoxfahCgldYDCgXAUXsC3eBkVGqLrqU759+1y/sw3S+vJzZ2Vwpt8lB7+0LEaJ7gLNnGD+x2LekvO+A==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 dev: false /@luma.gl/constants/8.5.14: @@ -2434,7 +2497,7 @@ packages: /@luma.gl/core/8.5.14: resolution: {integrity: sha512-YgXxWzBNnoIuo6BTsJdK9tfFBWSzmhqeCGsbGMOVimSRXNMDBXATDL5c/J8pkVSbphjhk5aSOmhUC7C5WzKKYQ==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@luma.gl/constants': 8.5.14 '@luma.gl/engine': 8.5.14 '@luma.gl/gltools': 8.5.14 @@ -2445,7 +2508,7 @@ packages: /@luma.gl/engine/8.5.14: resolution: {integrity: sha512-Dr05gfmHo2Zla0bejxaw+Zhjo1OwzGuUwwzXg/M4TE5U8YyC+mUMtIKcFILfMMEZxhZuWPU6jTDhvfHTIp+3xA==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@luma.gl/constants': 8.5.14 '@luma.gl/gltools': 8.5.14 '@luma.gl/shadertools': 8.5.14 @@ -2480,7 +2543,7 @@ packages: /@luma.gl/gltools/8.5.14: resolution: {integrity: sha512-ra8fSpkvIdaaBpD/zNryVYgYDRGNhStACIavM5wB5Gp3KVBbvUZWRl8a8YxRnv6MLWvBoEXnX7Dl7NmfWp6FPA==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@luma.gl/constants': 8.5.14 '@probe.gl/env': 3.5.0 '@types/offscreencanvas': 2019.7.0 @@ -2490,14 +2553,14 @@ packages: /@luma.gl/shadertools/8.5.14: resolution: {integrity: sha512-vq0XzMYzGM/A/usNvCzfWJw+ycN2vzqN4AtgrejrOrYu+6yP8LmfRp/iAxjhFkukCPgs03tUFYSlkY7I6HIVtw==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@math.gl/core': 3.6.0 dev: false /@luma.gl/webgl/8.5.14: resolution: {integrity: sha512-8FT9bm6k6XktETcL80AxmLGm27xWbQJYWjsHYjB4cP+PdePNnNQXNYrDN0WX7jKLECSgWIcaQ38O7itD9jACCw==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@luma.gl/constants': 8.5.14 '@luma.gl/gltools': 8.5.14 '@probe.gl/env': 3.5.0 @@ -2525,7 +2588,7 @@ packages: /@math.gl/core/3.6.0: resolution: {integrity: sha512-ZDgNC8iuP2qnG84kjiNKszugKvNHdp5iiiTQUwRPNxOY3qQ7Tw0WfoppCdUfrUdGaa7HEOXbnbJoy9Qnf/M1uQ==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@math.gl/types': 3.6.0 gl-matrix: 3.4.3 dev: false @@ -2533,7 +2596,7 @@ packages: /@math.gl/culling/3.6.0: resolution: {integrity: sha512-IBDQV3k0D54q3vHdAyJkdiTsZJIaWiPczHF1hfykeM5Lj3jtPmpyyhZMsea8yQeeJeacZ7//ZuMMUeqrEuhdxA==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@math.gl/core': 3.6.0 gl-matrix: 3.4.3 dev: false @@ -2541,7 +2604,7 @@ packages: /@math.gl/geospatial/3.6.0: resolution: {integrity: sha512-YSTBOuRksW9lO0BrfFbdfwBbu5SXalfTVni28LLIYsD3u+DYhYnQ/27PYpyfsyTJ364un4zmCo6DLCyjFUpOzA==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@math.gl/core': 3.6.0 gl-matrix: 3.4.3 dev: false @@ -2559,7 +2622,7 @@ packages: /@math.gl/web-mercator/3.6.0: resolution: {integrity: sha512-C9X1NUd6pA3G2g8CFDhi2a1UzsjFNljHl6a1cZ0qQkGMCpjRctHYpjQA1fJ/A+r7vgAxXvMf3EcY5odDOHeS6A==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 gl-matrix: 3.4.3 dev: false @@ -2595,20 +2658,20 @@ packages: /@probe.gl/env/3.5.0: resolution: {integrity: sha512-YdlpZZshhyYxvWDBmZ5RIW2pTR14Pw4p9czMlt/v7F6HbFzWfAdmH7q6xVwFRYxUpQLwhWensWyv4aFysiWl4g==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 dev: false /@probe.gl/log/3.5.0: resolution: {integrity: sha512-nW/qz2X1xY08WU/TsmJP6/6IPNcaY5fS/vLjpC4ahJuE2Mezga4hGM/R2X5JWE/nkPc+BsC5GnAnD13rwAxS7g==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@probe.gl/env': 3.5.0 dev: false /@probe.gl/stats/3.5.0: resolution: {integrity: sha512-IH2M+F3c8HR1DTroBARePUFG7wIewumtKA0UFqx51Z7S4hKrD60wFbpMmg0AcF4FvHAXMBoC+kYi1UKW9XbAOw==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 dev: false /@react-dnd/asap/4.0.1: @@ -2626,7 +2689,7 @@ packages: /@redux-saga/core/1.1.3: resolution: {integrity: sha512-8tInBftak8TPzE6X13ABmEtRJGjtK17w7VUs7qV17S8hCO5S3+aUTWZ/DBsBJPdE8Z5jOPwYALyvofgq1Ws+kg==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@redux-saga/deferred': 1.1.2 '@redux-saga/delay-p': 1.1.2 '@redux-saga/is': 1.1.2 @@ -2680,12 +2743,9 @@ packages: reselect: 4.1.5 dev: false - /@remix-run/router/0.1.0: - resolution: {integrity: sha512-mK2wZWAFsijTGIk7ZhKvEAzU7qIS6F6RhPy5AiEosaEs3mB3J2NvUWRoN3W74V+5Jju/AisT5NaM2pIbnfGXWg==} - dev: false - - /@remix-run/router/0.2.0-pre.3: - resolution: {integrity: sha512-/1vRiMh8j31PBgLgB2lqcwDDEBEwsnucOouZ8GvJTRNHULkPOU3kvQ1ZUpt9IPd1lS5AVC93I9pVLnFhlNA4Mw==} + /@remix-run/router/1.0.3: + resolution: {integrity: sha512-ceuyTSs7PZ/tQqi19YZNBc5X7kj1f8p+4DIyrcIYFY9h+hd1OKm4RqtiWldR9eGEvIiJfsqwM4BsuCtRIuEw6Q==} + engines: {node: '>=14'} dev: false /@rooks/use-mutation-observer/4.11.2_react@17.0.2: @@ -2721,8 +2781,8 @@ packages: /@types/babel__core/7.1.19: resolution: {integrity: sha512-WEOTgRsbYkvA/KCsDwVEGkd7WAr1e3g31VHQ8zy5gul/V1qKullU/BU5I68X5v7V3GnB9eotmom4v5a5gjxorw==} dependencies: - '@babel/parser': 7.17.9 - '@babel/types': 7.17.0 + '@babel/parser': 7.20.1 + '@babel/types': 7.20.0 '@types/babel__generator': 7.6.2 '@types/babel__template': 7.4.0 '@types/babel__traverse': 7.11.1 @@ -2731,20 +2791,20 @@ packages: /@types/babel__generator/7.6.2: resolution: {integrity: sha512-MdSJnBjl+bdwkLskZ3NGFp9YcXGx5ggLpQQPqtgakVhsWK0hTtNYhjpZLlWQTviGTvF8at+Bvli3jV7faPdgeQ==} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 dev: true /@types/babel__template/7.4.0: resolution: {integrity: sha512-NTPErx4/FiPCGScH7foPyr+/1Dkzkni+rHiYHHoTjvwou7AQzJkNeD60A9CXRy+ZEN2B1bggmkTMCDb+Mv5k+A==} dependencies: - '@babel/parser': 7.17.9 - '@babel/types': 7.17.0 + '@babel/parser': 7.20.1 + '@babel/types': 7.20.0 dev: true /@types/babel__traverse/7.11.1: resolution: {integrity: sha512-Vs0hm0vPahPMYi9tDjtP66llufgO3ST16WXaSTtDGEl9cewAl3AibmxWw6TINOqHPT9z0uABKAYjT9jNSg4npw==} dependencies: - '@babel/types': 7.17.0 + '@babel/types': 7.20.0 dev: true /@types/d3-timer/2.0.0: @@ -3366,7 +3426,7 @@ packages: '@ant-design/colors': 6.0.0 '@ant-design/icons': 4.7.0_sfoxds7t5ydpegc3knd667wn6m '@ant-design/react-slick': 0.28.2_react@17.0.2 - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@ctrl/tinycolor': 3.4.0 classnames: 2.3.1 copy-to-clipboard: 3.3.1 @@ -3447,7 +3507,7 @@ packages: resolution: {integrity: sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==} engines: {node: '>=6.0'} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@babel/runtime-corejs3': 7.13.10 dev: true @@ -3535,18 +3595,18 @@ packages: resolution: {integrity: sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==} dev: true - /babel-jest/27.5.1_@babel+core@7.17.9: + /babel-jest/27.5.1_@babel+core@7.19.6: resolution: {integrity: sha512-cdQ5dXjGRd0IBRATiQ4mZGlGlRE8kJpjPOixdNRdT+m3UcNqmYWN6rK6nvtXYfY3D76cb8s/O1Ss8ea24PIwcg==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} peerDependencies: '@babel/core': ^7.8.0 dependencies: - '@babel/core': 7.17.9 + '@babel/core': 7.19.6 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 '@types/babel__core': 7.1.19 babel-plugin-istanbul: 6.1.1 - babel-preset-jest: 27.5.1_@babel+core@7.17.9 + babel-preset-jest: 27.5.1_@babel+core@7.19.6 chalk: 4.1.2 graceful-fs: 4.2.9 slash: 3.0.0 @@ -3554,14 +3614,14 @@ packages: - supports-color dev: true - /babel-loader/8.2.4_vs5hf2sl7hjtttp43d2vzw3qay: + /babel-loader/8.2.4_h3kngfo6sqnotbut4nntjyiezm: resolution: {integrity: sha512-8dytA3gcvPPPv4Grjhnt8b5IIiTcq/zeXOPk4iTYI0SVXcsmuGg7JtBRDp8S9X+gJfhQ8ektjXZlDu1Bb33U8A==} engines: {node: '>= 8.9'} peerDependencies: '@babel/core': ^7.0.0 webpack: '>=2' dependencies: - '@babel/core': 7.17.9 + '@babel/core': 7.19.6 find-cache-dir: 3.3.1 loader-utils: 2.0.0 make-dir: 3.1.0 @@ -3569,12 +3629,6 @@ packages: webpack: 5.72.0_webpack-cli@4.9.2 dev: true - /babel-plugin-dynamic-import-node/2.3.3: - resolution: {integrity: sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==} - dependencies: - object.assign: 4.1.2 - dev: true - /babel-plugin-import/1.13.5: resolution: {integrity: sha512-IkqnoV+ov1hdJVofly9pXRJmeDm9EtROfrc5i6eII0Hix2xMs5FEm8FG3ExMvazbnZBbgHIt6qdO8And6lCloQ==} dependencies: @@ -3585,7 +3639,7 @@ packages: resolution: {integrity: sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==} engines: {node: '>=8'} dependencies: - '@babel/helper-plugin-utils': 7.17.12 + '@babel/helper-plugin-utils': 7.19.0 '@istanbuljs/load-nyc-config': 1.1.0 '@istanbuljs/schema': 0.1.3 istanbul-lib-instrument: 5.1.0 @@ -3598,44 +3652,44 @@ packages: resolution: {integrity: sha512-50wCwD5EMNW4aRpOwtqzyZHIewTYNxLA4nhB+09d8BIssfNfzBRhkBIHiaPv1Si226TQSvp8gxAJm2iY2qs2hQ==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@babel/template': 7.16.7 - '@babel/types': 7.17.0 + '@babel/template': 7.18.10 + '@babel/types': 7.20.0 '@types/babel__core': 7.1.19 '@types/babel__traverse': 7.11.1 dev: true - /babel-plugin-polyfill-corejs2/0.3.1_@babel+core@7.17.9: - resolution: {integrity: sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w==} + /babel-plugin-polyfill-corejs2/0.3.3_@babel+core@7.19.6: + resolution: {integrity: sha512-8hOdmFYFSZhqg2C/JgLUQ+t52o5nirNwaWM2B9LWteozwIvM14VSwdsCAUET10qT+kmySAlseadmfeeSWFCy+Q==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/compat-data': 7.17.7 - '@babel/core': 7.17.9 - '@babel/helper-define-polyfill-provider': 0.3.1_@babel+core@7.17.9 + '@babel/compat-data': 7.20.1 + '@babel/core': 7.19.6 + '@babel/helper-define-polyfill-provider': 0.3.3_@babel+core@7.19.6 semver: 6.3.0 transitivePeerDependencies: - supports-color dev: true - /babel-plugin-polyfill-corejs3/0.5.2_@babel+core@7.17.9: - resolution: {integrity: sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ==} + /babel-plugin-polyfill-corejs3/0.6.0_@babel+core@7.19.6: + resolution: {integrity: sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-define-polyfill-provider': 0.3.1_@babel+core@7.17.9 - core-js-compat: 3.21.0 + '@babel/core': 7.19.6 + '@babel/helper-define-polyfill-provider': 0.3.3_@babel+core@7.19.6 + core-js-compat: 3.26.0 transitivePeerDependencies: - supports-color dev: true - /babel-plugin-polyfill-regenerator/0.3.1_@babel+core@7.17.9: - resolution: {integrity: sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A==} + /babel-plugin-polyfill-regenerator/0.4.1_@babel+core@7.19.6: + resolution: {integrity: sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.17.9 - '@babel/helper-define-polyfill-provider': 0.3.1_@babel+core@7.17.9 + '@babel/core': 7.19.6 + '@babel/helper-define-polyfill-provider': 0.3.3_@babel+core@7.19.6 transitivePeerDependencies: - supports-color dev: true @@ -3645,8 +3699,8 @@ packages: peerDependencies: styled-components: '>= 2' dependencies: - '@babel/helper-annotate-as-pure': 7.16.7 - '@babel/helper-module-imports': 7.16.7 + '@babel/helper-annotate-as-pure': 7.18.6 + '@babel/helper-module-imports': 7.18.6 babel-plugin-syntax-jsx: 6.18.0 lodash: 4.17.21 picomatch: 2.3.1 @@ -3657,35 +3711,35 @@ packages: resolution: {integrity: sha512-qrPaCSo9c8RHNRHIotaufGbuOBN8rtdC4QrrFFc43vyWCCz7Kl7GL1PGaXtMGQZUXrkCjNEgxDfmAuAabr/rlw==} dev: false - /babel-preset-current-node-syntax/1.0.1_@babel+core@7.17.9: + /babel-preset-current-node-syntax/1.0.1_@babel+core@7.19.6: resolution: {integrity: sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ==} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.17.9 - '@babel/plugin-syntax-async-generators': 7.8.4_@babel+core@7.17.9 - '@babel/plugin-syntax-bigint': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-class-properties': 7.12.13_@babel+core@7.17.9 - '@babel/plugin-syntax-import-meta': 7.10.4_@babel+core@7.17.9 - '@babel/plugin-syntax-json-strings': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-logical-assignment-operators': 7.10.4_@babel+core@7.17.9 - '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-numeric-separator': 7.10.4_@babel+core@7.17.9 - '@babel/plugin-syntax-object-rest-spread': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-optional-catch-binding': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-optional-chaining': 7.8.3_@babel+core@7.17.9 - '@babel/plugin-syntax-top-level-await': 7.14.5_@babel+core@7.17.9 - dev: true - - /babel-preset-jest/27.5.1_@babel+core@7.17.9: + '@babel/core': 7.19.6 + '@babel/plugin-syntax-async-generators': 7.8.4_@babel+core@7.19.6 + '@babel/plugin-syntax-bigint': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-class-properties': 7.12.13_@babel+core@7.19.6 + '@babel/plugin-syntax-import-meta': 7.10.4_@babel+core@7.19.6 + '@babel/plugin-syntax-json-strings': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4_@babel+core@7.19.6 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-numeric-separator': 7.10.4_@babel+core@7.19.6 + '@babel/plugin-syntax-object-rest-spread': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-optional-chaining': 7.8.3_@babel+core@7.19.6 + '@babel/plugin-syntax-top-level-await': 7.14.5_@babel+core@7.19.6 + dev: true + + /babel-preset-jest/27.5.1_@babel+core@7.19.6: resolution: {integrity: sha512-Nptf2FzlPCWYuJg41HBqXVT8ym6bXOevuCTbhxlUpjwtysGaIWFvDEjp4y+G7fl13FgOdjs7P/DmErqH7da0Ag==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.17.9 + '@babel/core': 7.19.6 babel-plugin-jest-hoist: 27.5.1 - babel-preset-current-node-syntax: 1.0.1_@babel+core@7.17.9 + babel-preset-current-node-syntax: 1.0.1_@babel+core@7.19.6 dev: true /bail/2.0.2: @@ -3744,6 +3798,18 @@ packages: escalade: 3.1.1 node-releases: 2.0.3 picocolors: 1.0.0 + dev: true + + /browserslist/4.21.4: + resolution: {integrity: sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + dependencies: + caniuse-lite: 1.0.30001430 + electron-to-chromium: 1.4.284 + node-releases: 2.0.6 + update-browserslist-db: 1.0.10_browserslist@4.21.4 + dev: true /bser/2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} @@ -3783,14 +3849,19 @@ packages: /caniuse-api/3.0.0: resolution: {integrity: sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==} dependencies: - browserslist: 4.20.2 - caniuse-lite: 1.0.30001332 + browserslist: 4.21.4 + caniuse-lite: 1.0.30001430 lodash.memoize: 4.1.2 lodash.uniq: 4.5.0 dev: true /caniuse-lite/1.0.30001332: resolution: {integrity: sha512-10T30NYOEQtN6C11YGg411yebhvpnC6Z102+B95eAsN0oB6KUs01ivE8u+G6FMIRtIrVlYXhL+LUwQ3/hXwDWw==} + dev: true + + /caniuse-lite/1.0.30001430: + resolution: {integrity: sha512-IB1BXTZKPDVPM7cnV4iaKaHxckvdr/3xtctB3f7Hmenx3qYBhGtTZ//7EllK66aKXW98Lx0+7Yr0kxBtIt3tzg==} + dev: true /cartocolor/4.0.2: resolution: {integrity: sha512-+Gh9mb6lFxsDOLQlBLPxAHCnWXlg2W8q3AcVwqRcy95TdBbcOU89Wrb6h2Hd/6Ww1Kc1pzXmUdpnWD+xeCG0dg==} @@ -3975,6 +4046,7 @@ packages: resolution: {integrity: sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==} dependencies: safe-buffer: 5.1.2 + dev: true /copy-anything/2.0.3: resolution: {integrity: sha512-GK6QUtisv4fNS+XcI7shX0Gx9ORg7QqIznyfho79JTnX1XhLiyZHfftvGiziqzRiEi/Bjhgpi+D2o7HxJFPnDQ==} @@ -3988,11 +4060,10 @@ packages: toggle-selection: 1.0.6 dev: false - /core-js-compat/3.21.0: - resolution: {integrity: sha512-OSXseNPSK2OPJa6GdtkMz/XxeXx8/CJvfhQWTqd6neuUraujcL4jVsjkLQz1OWnax8xVQJnRPe0V2jqNWORA+A==} + /core-js-compat/3.26.0: + resolution: {integrity: sha512-piOX9Go+Z4f9ZiBFLnZ5VrOpBl0h7IGCkiFUN11QTe6LjAvOT3ifL/5TdoizMh99hcGy5SoLyWbapIY/PIb/3A==} dependencies: - browserslist: 4.20.2 - semver: 7.0.0 + browserslist: 4.21.4 dev: true /core-js-pure/3.9.1: @@ -4594,6 +4665,11 @@ packages: /electron-to-chromium/1.4.107: resolution: {integrity: sha512-Huen6taaVrUrSy8o7mGStByba8PfOWWluHNxSHGBrCgEdFVLtvdQDBr9LBCF9Uci8SYxh28QNNMO0oC17wbGAg==} + dev: true + + /electron-to-chromium/1.4.284: + resolution: {integrity: sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==} + dev: true /emittery/0.8.1: resolution: {integrity: sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg==} @@ -4688,6 +4764,7 @@ packages: /escalade/3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} + dev: true /escape-string-regexp/1.0.5: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} @@ -4731,7 +4808,7 @@ packages: peerDependencies: eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 aria-query: 4.2.2 array-includes: 3.1.4 ast-types-flow: 0.0.7 @@ -5145,6 +5222,7 @@ packages: /gensync/1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} + dev: true /get-caller-file/2.0.5: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} @@ -5586,8 +5664,8 @@ packages: resolution: {integrity: sha512-czwUz525rkOFDJxfKK6mYfIs9zBKILyrZQxjz3ABhjQXhbhFsSbo1HW/BFcsDnfJYJWA6thRR5/TUY2qs5W99Q==} engines: {node: '>=8'} dependencies: - '@babel/core': 7.17.9 - '@babel/parser': 7.17.9 + '@babel/core': 7.19.6 + '@babel/parser': 7.20.1 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.0 semver: 6.3.0 @@ -5698,10 +5776,10 @@ packages: ts-node: optional: true dependencies: - '@babel/core': 7.17.9 + '@babel/core': 7.19.6 '@jest/test-sequencer': 27.5.1 '@jest/types': 27.5.1 - babel-jest: 27.5.1_@babel+core@7.17.9 + babel-jest: 27.5.1_@babel+core@7.19.6 chalk: 4.1.2 ci-info: 3.3.0 deepmerge: 4.2.2 @@ -5859,7 +5937,7 @@ packages: resolution: {integrity: sha512-rMyFe1+jnyAAf+NHwTclDz0eAaLkVDdKVHHBFWsBWHnnh5YeJMNWWsv7AbFYXfK3oTqvL7VTWkhNLu1jX24D+g==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@babel/code-frame': 7.16.7 + '@babel/code-frame': 7.18.6 '@jest/types': 27.5.1 '@types/stack-utils': 2.0.0 chalk: 4.1.2 @@ -5996,16 +6074,16 @@ packages: resolution: {integrity: sha512-yYykXI5a0I31xX67mgeLw1DZ0bJB+gpq5IpSuCAoyDi0+BhgU/RIrL+RTzDmkNTchvDFWKP8lp+w/42Z3us5sA==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@babel/core': 7.17.9 - '@babel/generator': 7.17.9 - '@babel/plugin-syntax-typescript': 7.17.12_@babel+core@7.17.9 - '@babel/traverse': 7.17.9 - '@babel/types': 7.17.0 + '@babel/core': 7.19.6 + '@babel/generator': 7.20.1 + '@babel/plugin-syntax-typescript': 7.20.0_@babel+core@7.19.6 + '@babel/traverse': 7.20.1 + '@babel/types': 7.20.0 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 '@types/babel__traverse': 7.11.1 '@types/prettier': 2.6.0 - babel-preset-current-node-syntax: 1.0.1_@babel+core@7.17.9 + babel-preset-current-node-syntax: 1.0.1_@babel+core@7.19.6 chalk: 4.1.2 expect: 27.5.1 graceful-fs: 4.2.9 @@ -6211,6 +6289,7 @@ packages: resolution: {integrity: sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==} engines: {node: '>=6'} hasBin: true + dev: true /jsx-ast-utils/3.2.1: resolution: {integrity: sha512-uP5vu8xfy2F9A6LGC22KO7e2/vGTS1MhP+18f++ZNlf0Ohaxbc9nIEwHAsejlJKyzfZzU5UIhe5ItYkitcZnZA==} @@ -6790,6 +6869,11 @@ packages: /node-releases/2.0.3: resolution: {integrity: sha512-maHFz6OLqYxz+VQyCAtA3PTX4UP/53pa05fyDNc9CwjvJ0yEh6+xBwKsgCxMNhS8taUKBFYxfuiaD9U/55iFaw==} + dev: true + + /node-releases/2.0.6: + resolution: {integrity: sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==} + dev: true /normalize-path/3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} @@ -6953,7 +7037,7 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} dependencies: - '@babel/code-frame': 7.16.7 + '@babel/code-frame': 7.18.6 error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.1.6 @@ -7095,7 +7179,7 @@ packages: peerDependencies: postcss: ^8.2.15 dependencies: - browserslist: 4.20.2 + browserslist: 4.21.4 caniuse-api: 3.0.0 colord: 2.9.2 postcss: 8.4.12 @@ -7335,7 +7419,7 @@ packages: peerDependencies: postcss: ^8.2.15 dependencies: - browserslist: 4.20.2 + browserslist: 4.21.4 caniuse-api: 3.0.0 cssnano-utils: 3.1.0_postcss@8.4.12 postcss: 8.4.12 @@ -7370,7 +7454,7 @@ packages: peerDependencies: postcss: ^8.2.15 dependencies: - browserslist: 4.20.2 + browserslist: 4.21.4 cssnano-utils: 3.1.0_postcss@8.4.12 postcss: 8.4.12 postcss-value-parser: 4.2.0 @@ -7512,7 +7596,7 @@ packages: peerDependencies: postcss: ^8.2.15 dependencies: - browserslist: 4.20.2 + browserslist: 4.21.4 postcss: 8.4.12 postcss-value-parser: 4.2.0 dev: true @@ -7649,7 +7733,7 @@ packages: peerDependencies: postcss: ^8.2.15 dependencies: - browserslist: 4.20.2 + browserslist: 4.21.4 caniuse-api: 3.0.0 postcss: 8.4.12 dev: true @@ -7755,7 +7839,7 @@ packages: /probe.gl/3.5.0: resolution: {integrity: sha512-KWj8u0PNytr/rVwcQFcN7O8SK7n/ITOsUZ91l4fSX95oHhKvVCI7eadrzFUzFRlXkFfBWpMWZXFHITsHHHUctw==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@probe.gl/env': 3.5.0 '@probe.gl/log': 3.5.0 '@probe.gl/stats': 3.5.0 @@ -7841,7 +7925,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 dom-align: 1.12.0 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -7856,7 +7940,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 array-tree-filter: 2.1.0 classnames: 2.3.1 rc-select: 14.0.3_sfoxds7t5ydpegc3knd667wn6m @@ -7872,7 +7956,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 react: 17.0.2 react-dom: 17.0.2_react@17.0.2 @@ -7884,7 +7968,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -7899,7 +7983,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -7913,7 +7997,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -7926,7 +8010,7 @@ packages: react: '*' react-dom: '*' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-trigger: 5.2.10_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -7940,7 +8024,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 async-validator: 4.0.7 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -7953,7 +8037,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-dialog: 8.6.0_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -7967,7 +8051,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -7980,7 +8064,7 @@ packages: react: '>=16.0.0' react-dom: '>=16.0.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -7993,7 +8077,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-menu: 9.3.2_sfoxds7t5ydpegc3knd667wn6m rc-textarea: 0.3.4_sfoxds7t5ydpegc3knd667wn6m @@ -8009,7 +8093,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m rc-overflow: 1.2.2_sfoxds7t5ydpegc3knd667wn6m @@ -8026,7 +8110,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8040,7 +8124,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -8054,7 +8138,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-resize-observer: 1.2.0_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -8068,7 +8152,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 react: 17.0.2 react-dom: 17.0.2_react@17.0.2 @@ -8081,7 +8165,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 date-fns: 2.28.0 dayjs: 1.11.1 @@ -8099,7 +8183,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8113,7 +8197,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8126,7 +8210,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8141,7 +8225,7 @@ packages: react: '*' react-dom: '*' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m rc-overflow: 1.2.2_sfoxds7t5ydpegc3knd667wn6m @@ -8159,7 +8243,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-tooltip: 5.1.1_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -8175,7 +8259,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8188,7 +8272,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8202,7 +8286,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-resize-observer: 1.2.0_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -8218,7 +8302,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-dropdown: 3.3.2_sfoxds7t5ydpegc3knd667wn6m rc-menu: 9.3.2_sfoxds7t5ydpegc3knd667wn6m @@ -8234,7 +8318,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-resize-observer: 1.2.0_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -8248,7 +8332,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 rc-trigger: 5.2.10_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 react-dom: 17.0.2_react@17.0.2 @@ -8260,7 +8344,7 @@ packages: react: '*' react-dom: '*' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-select: 14.0.3_sfoxds7t5ydpegc3knd667wn6m rc-tree: 5.4.4_sfoxds7t5ydpegc3knd667wn6m @@ -8276,7 +8360,7 @@ packages: react: '*' react-dom: '*' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m @@ -8292,7 +8376,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-align: 4.0.9_sfoxds7t5ydpegc3knd667wn6m rc-motion: 2.4.5_sfoxds7t5ydpegc3knd667wn6m @@ -8307,7 +8391,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 classnames: 2.3.1 rc-util: 5.20.1_sfoxds7t5ydpegc3knd667wn6m react: 17.0.2 @@ -8320,7 +8404,7 @@ packages: react: '>=16.9.0' react-dom: '>=16.9.0' dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 react: 17.0.2 react-dom: 17.0.2_react@17.0.2 react-is: 16.13.1 @@ -8346,7 +8430,7 @@ packages: peerDependencies: react: ^15.3.0 || ^16.0.0 || ^17.0.0 dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 react: 17.0.2 dev: false @@ -8410,7 +8494,7 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 focus-lock: 0.9.2 prop-types: 15.8.1 react: 17.0.2 @@ -8489,7 +8573,7 @@ packages: react-native: optional: true dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 '@types/react-redux': 7.1.24 hoist-non-react-statics: 3.3.2 loose-envify: 1.4.0 @@ -8499,23 +8583,26 @@ packages: react-is: 17.0.2 dev: false - /react-router-dom/6.4.0-pre.8_sfoxds7t5ydpegc3knd667wn6m: - resolution: {integrity: sha512-5hxQdiPOG98YOANbJ8IBUKbRgU6k4/TSX4VyO6jRWoG6KY0K2EG5wjEZgZ0tlAN9fXFgnHsro/HKi/r+HxugAg==} + /react-router-dom/6.4.3_sfoxds7t5ydpegc3knd667wn6m: + resolution: {integrity: sha512-MiaYQU8CwVCaOfJdYvt84KQNjT78VF0TJrA17SIQgNHRvLnXDJO6qsFqq8F/zzB1BWZjCFIrQpu4QxcshitziQ==} + engines: {node: '>=14'} peerDependencies: react: '>=16.8' react-dom: '>=16.8' dependencies: + '@remix-run/router': 1.0.3 react: 17.0.2 react-dom: 17.0.2_react@17.0.2 - react-router: 6.4.0-pre.8_react@17.0.2 + react-router: 6.4.3_react@17.0.2 dev: false - /react-router/6.4.0-pre.8_react@17.0.2: - resolution: {integrity: sha512-xwt2s9jW5NaakymlSRG0hZoGda6E/Skfgmj/4vlwvRX/kqoO8jU0g+Vi3sHDCatH/V+GjDyTQkdMEDm8qGsBsQ==} + /react-router/6.4.3_react@17.0.2: + resolution: {integrity: sha512-BT6DoGn6aV1FVP5yfODMOiieakp3z46P1Fk0RNzJMACzE7C339sFuHebfvWtnB4pzBvXXkHP2vscJzWRuUjTtA==} + engines: {node: '>=14'} peerDependencies: react: '>=16.8' dependencies: - '@remix-run/router': 0.2.0-pre.3 + '@remix-run/router': 1.0.3 react: 17.0.2 dev: false @@ -8537,7 +8624,7 @@ packages: react: ^15.0.0 || ^16.0.0 || ^17.0.0 react-dom: ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 memoize-one: 5.1.1 react: 17.0.2 react-dom: 17.0.2_react@17.0.2 @@ -8611,10 +8698,10 @@ packages: /redux/4.1.2: resolution: {integrity: sha512-SH8PglcebESbd/shgf6mii6EIoRM0zrQyjcuQ+ojmfxjTtE0z9Y8pa62iA/OJ58qjP6j27uyW4kUF4jl/jd6sw==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 - /regenerate-unicode-properties/10.0.1: - resolution: {integrity: sha512-vn5DU6yg6h8hP/2OkQo3K7uVILvY4iu0oI4t3HFa81UPkhGJwkRwM10JEc3upjdhHjs/k8GJY1sRBhk5sr69Bw==} + /regenerate-unicode-properties/10.1.0: + resolution: {integrity: sha512-d1VudCLoIGitcU/hEg2QqvyGZQmdC0Lf8BqdOMXGFSvJP4bNV1+XqbPQeHHLD51Jh4QJJ225dlIFvY4Ly6MXmQ==} engines: {node: '>=4'} dependencies: regenerate: 1.4.2 @@ -8624,13 +8711,13 @@ packages: resolution: {integrity: sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==} dev: true - /regenerator-runtime/0.13.8: - resolution: {integrity: sha512-o/ASGwgZ6UiVjspr4YnzHKF1NbBdX+mCPkSeymofk/d7I+csCYn3ZgZMMVtXeecpT8DBiI2nAlYkHd+xNCqu4A==} + /regenerator-runtime/0.13.10: + resolution: {integrity: sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==} - /regenerator-transform/0.14.5: - resolution: {integrity: sha512-eOf6vka5IO151Jfsw2NO9WpGX58W6wWmefK3I1zEGr0lOD0u8rwPaNqQL1aRxUaxLeKO3ArNh3VYg1KbaD+FFw==} + /regenerator-transform/0.15.0: + resolution: {integrity: sha512-LsrGtPmbYg19bcPHwdtmXwbW+TqNvtY4riE3P83foeHRroMbH6/2ddFBfab3t7kbzc7v7p4wbkIecHImqt0QNg==} dependencies: - '@babel/runtime': 7.17.9 + '@babel/runtime': 7.20.1 dev: true /regexp.prototype.flags/1.3.1: @@ -8646,24 +8733,24 @@ packages: engines: {node: '>=8'} dev: true - /regexpu-core/5.0.1: - resolution: {integrity: sha512-CriEZlrKK9VJw/xQGJpQM5rY88BtuL8DM+AEwvcThHilbxiTAy8vq4iJnd2tqq8wLmjbGZzP7ZcKFjbGkmEFrw==} + /regexpu-core/5.2.1: + resolution: {integrity: sha512-HrnlNtpvqP1Xkb28tMhBUO2EbyUHdQlsnlAhzWcwHy8WJR53UWr7/MAvqrsQKMbV4qdpv03oTMG8iIhfsPFktQ==} engines: {node: '>=4'} dependencies: regenerate: 1.4.2 - regenerate-unicode-properties: 10.0.1 - regjsgen: 0.6.0 - regjsparser: 0.8.4 + regenerate-unicode-properties: 10.1.0 + regjsgen: 0.7.1 + regjsparser: 0.9.1 unicode-match-property-ecmascript: 2.0.0 unicode-match-property-value-ecmascript: 2.0.0 dev: true - /regjsgen/0.6.0: - resolution: {integrity: sha512-ozE883Uigtqj3bx7OhL1KNbCzGyW2NQZPl6Hs09WTvCuZD5sTI4JY58bkbQWa/Y9hxIsvJ3M8Nbf7j54IqeZbA==} + /regjsgen/0.7.1: + resolution: {integrity: sha512-RAt+8H2ZEzHeYWxZ3H2z6tF18zyyOnlcdaafLrm21Bguj7uZy6ULibiAFdXEtKQY4Sy7wDTwDiOazasMLc4KPA==} dev: true - /regjsparser/0.8.4: - resolution: {integrity: sha512-J3LABycON/VNEu3abOviqGHuB/LOtOQj8SKmfP9anY5GfAVw/SPjwzSjxGjbZXIxbGfqTHtJw58C2Li/WkStmA==} + /regjsparser/0.9.1: + resolution: {integrity: sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ==} hasBin: true dependencies: jsesc: 0.5.0 @@ -8779,6 +8866,7 @@ packages: /safe-buffer/5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: true /safe-buffer/5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -8861,10 +8949,6 @@ packages: /semver/6.3.0: resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==} hasBin: true - - /semver/7.0.0: - resolution: {integrity: sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==} - hasBin: true dev: true /semver/7.3.7: @@ -8939,10 +9023,6 @@ packages: source-map: 0.6.1 dev: true - /source-map/0.5.7: - resolution: {integrity: sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==} - engines: {node: '>=0.10.0'} - /source-map/0.6.1: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} @@ -9095,7 +9175,7 @@ packages: peerDependencies: postcss: ^8.2.15 dependencies: - browserslist: 4.20.2 + browserslist: 4.21.4 postcss: 8.4.12 postcss-selector-parser: 6.0.10 dev: true @@ -9476,6 +9556,17 @@ packages: engines: {node: '>= 4.0.0'} dev: true + /update-browserslist-db/1.0.10_browserslist@4.21.4: + resolution: {integrity: sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + dependencies: + browserslist: 4.21.4 + escalade: 3.1.1 + picocolors: 1.0.0 + dev: true + /uri-js/4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} dependencies: diff --git a/src/main/webapp/resources/js/pages/projects/ProjectSPA.tsx b/src/main/webapp/resources/js/pages/projects/ProjectSPA.tsx index 5f4f026d8cb..6e4abf450fd 100644 --- a/src/main/webapp/resources/js/pages/projects/ProjectSPA.tsx +++ b/src/main/webapp/resources/js/pages/projects/ProjectSPA.tsx @@ -1,6 +1,12 @@ import React, { Suspense } from "react"; import { render } from "react-dom"; -import { DataBrowserRouter, Outlet, Route } from "react-router-dom"; +import { + createBrowserRouter, + createRoutesFromElements, + Outlet, + Route, + RouterProvider, +} from "react-router-dom"; import { loader as exportsLoader } from "../../components/ncbi/export-table"; import { setBaseUrl } from "../../utilities/url-utilities"; import { loader as detailsLoader } from "../../components/ncbi/details"; @@ -21,6 +27,37 @@ const DefaultErrorBoundary = React.lazy( __webpack_public_path__ = setBaseUrl(`/dist/`); +const router = createBrowserRouter( + createRoutesFromElements( + }> + } + loader={ncbiLoader} + errorElement={} + /> + } + errorElement={} + > + } + loader={exportsLoader} + errorElement={} + /> + } + loader={detailsLoader} + errorElement={} + /> + + + ) +); + /** * Default layout for the Project Single Page Application * @constructor @@ -43,39 +80,7 @@ function ProjectBase(): JSX.Element { * @constructor */ export default function ProjectSPA(): JSX.Element { - return ( - - } - > - } - loader={ncbiLoader} - errorElement={} - /> - } - errorElement={} - > - } - loader={exportsLoader} - errorElement={} - /> - } - loader={detailsLoader} - errorElement={} - /> - - - - ); + return ; } render(, document.querySelector("#root")); From d5030439ecab48f0dd0b0340bcef90c8f252184a Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 4 Nov 2022 08:20:26 -0500 Subject: [PATCH 357/655] chore: Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 819c5473138..d389f1198f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # Changelog ## [Unreleased] +* [UI/Developer]: Updated `react-router` to the version 6.4.3. See[PR 1405](https://github.com/phac-nml/irida/pull/1405) ## [22.09.1] - 2022/10/21 * [UI]: Fixed when sharing or exporting sample on the project sample page, and other minor bugs. See [PR 1382](https://github.com/phac-nml/irida/pull/1382) From 4db2639a4ab9b03e7115dbdbb9d5a23d170b66fa Mon Sep 17 00:00:00 2001 From: Eric Enns Date: Fri, 4 Nov 2022 08:39:53 -0500 Subject: [PATCH 358/655] feat(build): gradle buildWebapp now only runs when packages change, inputs change, or output directory is modified. --- build.gradle.kts | 14 ++- settings.gradle.kts | 2 +- src/main/webapp/package.json | 13 +- src/main/webapp/pnpm-lock.yaml | 219 +++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 12 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 9e81cfbe510..32bfc15525f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -304,15 +304,19 @@ tasks.register("cleanWebapp") { } tasks.register("buildWebapp") { - dependsOn(":cleanWebapp") - pnpmCommand.set(listOf("build")) - inputs.dir("${project.projectDir}/src/main/webapp/resources") + inputs.files(fileTree("${project.projectDir}/src/main/webapp/resources")) + inputs.file("${project.projectDir}/src/main/webapp/package.json") + inputs.file("${project.projectDir}/src/main/webapp/pnpm-lock.yaml") + outputs.dir("${project.projectDir}/src/main/webapp/dist") + + dependsOn(":pnpmInstall") + pnpmCommand.set(listOf("clean", "build")) } tasks.register("startWebapp") { - dependsOn(":cleanWebapp") - pnpmCommand.set(listOf("start")) + dependsOn(":pnpmInstall") + pnpmCommand.set(listOf("clean", "start")) inputs.dir("${project.projectDir}/src/main/webapp/resources") outputs.dir("${project.projectDir}/src/main/webapp/dist") } diff --git a/settings.gradle.kts b/settings.gradle.kts index de2c326a16c..5010e60c23d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,7 +4,7 @@ pluginManagement { gradlePluginPortal() } plugins { - id("com.github.node-gradle.node") version "3.4.0" + id("com.github.node-gradle.node") version "3.5.0" id("io.spring.dependency-management") version "1.0.11.RELEASE" id("org.gradle.test-retry") version "1.4.0" id("org.springframework.boot") version "2.7.4" diff --git a/src/main/webapp/package.json b/src/main/webapp/package.json index a7ff2001b73..c2d761131cd 100644 --- a/src/main/webapp/package.json +++ b/src/main/webapp/package.json @@ -3,12 +3,12 @@ "version": "1.0.0", "license": "Apache-2.0", "scripts": { - "build": "webpack --mode production", - "start": "webpack --mode development --watch", - "clean": "rm -rf dist/ pages/templates/i18n/", - "test": "jest resources/js/**/*.test.js", - "test_watch": "jest --watch resources/js/**/*.test.js", - "lint": "eslint --ext resources/js/**/*.{js,jsx,ts,tsx}" + "build": "run-z --then webpack --mode production", + "start": "run-z --then webpack --mode development --watch", + "clean": "run-z --then rm -rf dist/ pages/templates/i18n/", + "test": "run-z --then jest resources/js/**/*.test.js", + "test_watch": "run-z --then jest --watch resources/js/**/*.test.js", + "lint": "run-z --then eslint --ext resources/js/**/*.{js,jsx,ts,tsx}" }, "browserslist": [ "last 2 Chrome versions", @@ -128,6 +128,7 @@ "prettier": "^2.7.0", "properties-reader": "^2.2.0", "react-is": "^17.0.2", + "run-z": "^1.10.1", "speed-measure-webpack-plugin": "^1.5.0", "terser-webpack-plugin": "^5.3.1", "typescript": "^4.7.2", diff --git a/src/main/webapp/pnpm-lock.yaml b/src/main/webapp/pnpm-lock.yaml index 0e7d54ac308..c8651e76944 100644 --- a/src/main/webapp/pnpm-lock.yaml +++ b/src/main/webapp/pnpm-lock.yaml @@ -114,6 +114,7 @@ specifiers: reactour: ^1.18.7 redux: 4.1.2 redux-saga: ^1.1.3 + run-z: ^1.10.1 speed-measure-webpack-plugin: ^1.5.0 styled-components: ^5.3.5 terser-webpack-plugin: ^5.3.1 @@ -237,6 +238,7 @@ devDependencies: prettier: 2.7.1 properties-reader: 2.2.0 react-is: 17.0.2 + run-z: 1.10.1 speed-measure-webpack-plugin: 1.5.0_webpack@5.72.0 terser-webpack-plugin: 5.3.1_webpack@5.72.0 typescript: 4.7.2 @@ -2611,6 +2613,47 @@ packages: '@babel/runtime': 7.17.9 dev: false + /@proc7ts/context-values/7.0.1: + resolution: {integrity: sha512-I50cq8eTUQenWgeZ7thHxmsiW2uqez/SsOWofvvzzLklAWypA/2xpmVUUc29e0z6IwmCZrqHVUsbZVPkbzhyCw==} + peerDependencies: + '@proc7ts/fun-events': ^10.5.1 + peerDependenciesMeta: + '@proc7ts/fun-events': + optional: true + dependencies: + '@proc7ts/primitives': 3.0.2 + '@proc7ts/push-iterator': 3.1.0 + '@proc7ts/supply': 1.2.3 + transitivePeerDependencies: + - '@proc7ts/call-thru' + dev: true + + /@proc7ts/logger/1.3.2: + resolution: {integrity: sha512-hzPVA4GcdUYxdYd9RSG1pZ+VLa56HDIhhyqlofcGdyVJbIfazbpwwMR4ZC1zlFGHefvQ+RbjskBCOCeRTiGgqA==} + dependencies: + '@proc7ts/context-values': 7.0.1 + transitivePeerDependencies: + - '@proc7ts/call-thru' + - '@proc7ts/fun-events' + dev: true + + /@proc7ts/primitives/3.0.2: + resolution: {integrity: sha512-sPGz+vVXydw0wUP2eixVNvWzwrXsrW+j1NmCH3/LQo4tE1/jODQGkXB2MWTCQqNy3grpoAf6piOOpxAynm4tZw==} + dev: true + + /@proc7ts/push-iterator/3.1.0: + resolution: {integrity: sha512-5pgO8yaMMn1LnIMPMxtchSsqHHQsAEiMQ9EszOSijn4dwYADnPmGXh1hSG34EBHEIhhohZzxu/tN0oAAsSki+Q==} + peerDependencies: + '@proc7ts/call-thru': ^4.4.0 + peerDependenciesMeta: + '@proc7ts/call-thru': + optional: true + dev: true + + /@proc7ts/supply/1.2.3: + resolution: {integrity: sha512-NxIArWgpwyMKFaoklfEXkq3/0mLDl64ZrWxFFE9t/uBUgkzKnq3mSmw9gmeQQlnjkFS9mXInH873qBOOzzK5bw==} + dev: true + /@react-dnd/asap/4.0.1: resolution: {integrity: sha512-kLy0PJDDwvwwTXxqTFNAAllPHD73AycE9ypWeln/IguoGBEbvFcPDbCV03G52bEcC5E+YgupBE0VzHGdC8SIXg==} dev: false @@ -2696,6 +2739,38 @@ packages: react: 17.0.2 dev: false + /@run-z/exec-z/1.4.1: + resolution: {integrity: sha512-d05A1gfSPbHeqqg0IJxb4Bw2lZ6JA7J9GFL+7sXNkP4hbkcPQCGr+gmJFM4V5RcAmJgWh9kkgQT2CVVyJ13QZQ==} + dependencies: + '@proc7ts/primitives': 3.0.2 + dev: true + + /@run-z/log-z/2.1.0: + resolution: {integrity: sha512-+1oM4HtYi8xH+h3dxfAo9RenyPDtUfKAuSqrELK8AbtVwZ36652AXCqEmR3jJE4Jr9/LweTtlMpf50stUj8oEA==} + dependencies: + '@proc7ts/context-values': 7.0.1 + '@proc7ts/logger': 1.3.2 + '@proc7ts/primitives': 3.0.2 + '@proc7ts/push-iterator': 3.1.0 + transitivePeerDependencies: + - '@proc7ts/call-thru' + - '@proc7ts/fun-events' + dev: true + + /@run-z/optionz/2.4.0_chalk@5.1.2: + resolution: {integrity: sha512-GU8zFgmo5D1PbGJ1YI1hpNCQDTseo1iSjQd2R8ilINndoocvKtrLbYf1xvfv/vT6Q689UTMXjQC9aURE50uYkA==} + peerDependencies: + chalk: ^5.0.0 + peerDependenciesMeta: + chalk: + optional: true + dependencies: + '@proc7ts/primitives': 3.0.2 + chalk: 5.1.2 + string-width: 5.1.2 + wrap-ansi: 8.0.1 + dev: true + /@sinonjs/commons/1.8.2: resolution: {integrity: sha512-sruwd86RJHdsVf/AtBoijDmUqJp3B6hF/DGC23C+JaegnDHaZyewCjoVGTdg3J0uz3Zs7NnIT05OBOmML72lQw==} dependencies: @@ -3334,11 +3409,23 @@ packages: type-fest: 0.21.3 dev: true + /ansi-escapes/5.0.0: + resolution: {integrity: sha512-5GFMVX8HqE/TB+FuBJGuO5XG0WrsA6ptUqoODaT/n9mmUaZFkqnBueB4leqGBCmrUHnCnC4PCZTCd0E7QQ83bA==} + engines: {node: '>=12'} + dependencies: + type-fest: 1.4.0 + dev: true + /ansi-regex/5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} dev: true + /ansi-regex/6.0.1: + resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} + engines: {node: '>=12'} + dev: true + /ansi-styles/3.2.1: resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} engines: {node: '>=4'} @@ -3357,6 +3444,11 @@ packages: engines: {node: '>=10'} dev: true + /ansi-styles/6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + dev: true + /antd/4.19.5_sfoxds7t5ydpegc3knd667wn6m: resolution: {integrity: sha512-C4H/VJqlVO5iMvHZyiV27R8SbPs4jsOKCGPhDXIHUry/RnUCbMmVeQaPRfUIxSI1NbqDflsuQfevPtz1svyIlg==} peerDependencies: @@ -3822,6 +3914,11 @@ packages: supports-color: 7.2.0 dev: true + /chalk/5.1.2: + resolution: {integrity: sha512-E5CkT4jWURs1Vy5qGJye+XwCkNj7Od3Af7CP6SujMetSMkLs8Do2RWJK5yx1wamHV/op8Rz+9rltjaTQWDnEFQ==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + dev: true + /char-regex/1.0.2: resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} engines: {node: '>=10'} @@ -3850,6 +3947,19 @@ packages: resolution: {integrity: sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA==} dev: false + /cli-spinners/2.7.0: + resolution: {integrity: sha512-qu3pN8Y3qHNgE2AFweciB1IfMnmZ/fsNTEE+NOFjmGB2F/7rLhnhzppvpCnN4FovtP26k8lHyy9ptEbNwWFLzw==} + engines: {node: '>=6'} + dev: true + + /cli-truncate/3.1.0: + resolution: {integrity: sha512-wfOBkjXteqSnI59oPcJkcPl/ZmwvMMOj340qUIY1SKZCv0B9Cf4D4fAucRkIKQmsIuYK3x1rrgU7MeGRruiuiA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + slice-ansi: 5.0.0 + string-width: 5.1.2 + dev: true + /clipboard/1.7.1: resolution: {integrity: sha512-smkaRaIQsrnKN1F3wd1/vY9Q+DeR4L8ZCXKeHCFC2j8RZuSBbuImcLdnIO4GTxmzJxQuDGNKkyfpGoPW7Ua5bQ==} dependencies: @@ -4592,6 +4702,10 @@ packages: resolution: {integrity: sha512-iRDI1QeCQIhMCZk48DRDMVgQSSBDmbzzNhnxIo+pwx3swkfjMh6vh0nWLq1NdvGHLKH6wIrAM3vQWeTj6qeoug==} dev: false + /eastasianwidth/0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + dev: true + /electron-to-chromium/1.4.107: resolution: {integrity: sha512-Huen6taaVrUrSy8o7mGStByba8PfOWWluHNxSHGBrCgEdFVLtvdQDBr9LBCF9Uci8SYxh28QNNMO0oC17wbGAg==} @@ -5480,6 +5594,11 @@ packages: engines: {node: '>=8'} dev: true + /is-fullwidth-code-point/4.0.0: + resolution: {integrity: sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==} + engines: {node: '>=12'} + dev: true + /is-generator-fn/2.1.0: resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==} engines: {node: '>=6'} @@ -5558,6 +5677,11 @@ packages: resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} dev: true + /is-unicode-supported/1.3.0: + resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} + engines: {node: '>=12'} + dev: true + /is-weakref/1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -6377,6 +6501,14 @@ packages: /lodash/4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + /log-symbols/5.1.0: + resolution: {integrity: sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==} + engines: {node: '>=12'} + dependencies: + chalk: 5.1.2 + is-unicode-supported: 1.3.0 + dev: true + /long/3.2.0: resolution: {integrity: sha512-ZYvPPOMqUwPoDsbJaR10iQJYnMuZhRTvHYl62ErLIEX7RgFlziSBUUvrt3OVfc47QlHHpzPZYP17g3Fv7oeJkg==} engines: {node: '>=0.6'} @@ -6813,6 +6945,13 @@ packages: path-key: 3.1.1 dev: true + /npm-run-path/5.1.0: + resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + path-key: 4.0.0 + dev: true + /nth-check/2.0.1: resolution: {integrity: sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==} dependencies: @@ -6983,6 +7122,11 @@ packages: engines: {node: '>=8'} dev: true + /path-key/4.0.0: + resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} + engines: {node: '>=12'} + dev: true + /path-parse/1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} dev: true @@ -8766,6 +8910,34 @@ packages: queue-microtask: 1.2.3 dev: true + /run-z/1.10.1: + resolution: {integrity: sha512-URsE2xx6RK3iumCrQS7gbNcCj7ffNRWcytOmSh9ctIeDZjeZ22fbH47TTVR5BiEIcM6PuLe+sjRn5XCc1G2Dvg==} + engines: {node: '>=12'} + hasBin: true + dependencies: + '@proc7ts/logger': 1.3.2 + '@proc7ts/primitives': 3.0.2 + '@run-z/exec-z': 1.4.1 + '@run-z/log-z': 2.1.0 + '@run-z/optionz': 2.4.0_chalk@5.1.2 + ansi-escapes: 5.0.0 + chalk: 5.1.2 + cli-spinners: 2.7.0 + cli-truncate: 3.1.0 + cross-spawn: 7.0.3 + log-symbols: 5.1.0 + npm-run-path: 5.1.0 + path-key: 4.0.0 + semver: 7.3.7 + shell-quote: 1.7.4 + string-width: 5.1.2 + strip-ansi: 7.0.1 + tree-kill: 1.2.2 + transitivePeerDependencies: + - '@proc7ts/call-thru' + - '@proc7ts/fun-events' + dev: true + /rw/1.3.3: resolution: {integrity: sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==} dev: false @@ -8904,6 +9076,10 @@ packages: engines: {node: '>=8'} dev: true + /shell-quote/1.7.4: + resolution: {integrity: sha512-8o/QEhSSRb1a5i7TFR0iM4G16Z0vYB2OQVs4G3aAFXjn3T6yEx8AZxy1PgDF7I00LZHYA3WxaSYIf5e5sAX8Rw==} + dev: true + /side-channel/1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: @@ -8928,6 +9104,14 @@ packages: engines: {node: '>=8'} dev: true + /slice-ansi/5.0.0: + resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + is-fullwidth-code-point: 4.0.0 + dev: true + /source-map-js/1.0.2: resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} engines: {node: '>=0.10.0'} @@ -9010,6 +9194,15 @@ packages: strip-ansi: 6.0.1 dev: true + /string-width/5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.0.1 + dev: true + /string.prototype.matchall/4.0.6: resolution: {integrity: sha512-6WgDX8HmQqvEd7J+G6VtAahhsQIssiZ8zl7zKh1VDMFyL3hRTJP4FTNA3RbIp2TOQ9AYNDcc7e3fH0Qbup+DBg==} dependencies: @@ -9044,6 +9237,13 @@ packages: ansi-regex: 5.0.1 dev: true + /strip-ansi/7.0.1: + resolution: {integrity: sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==} + engines: {node: '>=12'} + dependencies: + ansi-regex: 6.0.1 + dev: true + /strip-bom/4.0.0: resolution: {integrity: sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==} engines: {node: '>=8'} @@ -9280,6 +9480,11 @@ packages: punycode: 2.1.1 dev: true + /tree-kill/1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + dev: true + /trough/2.1.0: resolution: {integrity: sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==} dev: false @@ -9329,6 +9534,11 @@ packages: engines: {node: '>=10'} dev: true + /type-fest/1.4.0: + resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} + engines: {node: '>=10'} + dev: true + /typedarray-to-buffer/3.1.5: resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} dependencies: @@ -9770,6 +9980,15 @@ packages: strip-ansi: 6.0.1 dev: true + /wrap-ansi/8.0.1: + resolution: {integrity: sha512-QFF+ufAqhoYHvoHdajT/Po7KoXVBPXS2bgjIam5isfWJPfIOnQZ50JtUiVvCv/sjgacf3yRrt2ZKUZ/V4itN4g==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.0.1 + dev: true + /wrappy/1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: true From 54a21faf3acf2e3531a067fd89bfe85d47b15e86 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 4 Nov 2022 09:28:54 -0500 Subject: [PATCH 359/655] chore: Get the user details --- .../resources/js/pages/search/index.tsx | 38 ++++++++----------- .../js/pages/search/loaders/user-loader.ts | 2 +- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index 590fd05ea6a..2b9e1263454 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -1,26 +1,26 @@ import { Checkbox, Input, Layout, PageHeader, Space } from "antd"; -import React, { useEffect, useRef } from "react"; +import React, { useRef } from "react"; import { render } from "react-dom"; import { - DataBrowserRouter, - Outlet, + createBrowserRouter, + createRoutesFromElements, Route, - Routes, + RouterProvider, useLoaderData, useSearchParams, } from "react-router-dom"; import { setBaseUrl } from "../../utilities/url-utilities"; import userLoader from "./loaders/user-loader"; -function SearchPage() { - return ( - - - } /> - - - ); -} +const router = createBrowserRouter( + createRoutesFromElements( + } + loader={userLoader} + /> + ) +); function SearchLayout() { const user = useLoaderData(); @@ -33,11 +33,11 @@ function SearchLayout() { - +
    Project Samples - Search Global + {user.admin && Search Global}
    @@ -48,13 +48,7 @@ function SearchLayout() { const element = document.querySelector("#root"); render( - - } - loader={userLoader} - /> - + , element ); diff --git a/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts b/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts index 48802c940cd..683a93c4479 100644 --- a/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts +++ b/src/main/webapp/resources/js/pages/search/loaders/user-loader.ts @@ -1,4 +1,4 @@ import { fetchCurrentUserDetails } from "../../../apis/users/user"; -export async function loader() { +export default async function loader() { return fetchCurrentUserDetails(); } From 2e25abb16d01af15f94ca7f4de5560bd0affd12e Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 4 Nov 2022 14:40:04 -0500 Subject: [PATCH 360/655] starting to create new batch api endpoints --- .../ria/web/ajax/dto/UpdateSampleRequest.java | 14 +++- .../ProjectSamplesAjaxController.java | 25 ++++--- .../web/services/UIProjectSampleService.java | 66 ++++++++++++++++--- .../resources/js/apis/projects/samples.ts | 15 ++--- .../services/importReducer.ts | 30 +++++---- .../services/UIProjectSampleServiceTest.java | 12 ++-- 6 files changed, 109 insertions(+), 53 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/UpdateSampleRequest.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/UpdateSampleRequest.java index 793a60130e9..216587167ef 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/UpdateSampleRequest.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/UpdateSampleRequest.java @@ -6,7 +6,19 @@ * UI Request to update an existing sample */ public class UpdateSampleRequest extends CreateSampleRequest { - public UpdateSampleRequest(String name, String organism, String description, List metadata) { + private Long sampleId; + + public UpdateSampleRequest(Long sampleID, String name, String organism, String description, + List metadata) { super(name, organism, description, metadata); + this.sampleId = sampleID; + } + + public Long getSampleId() { + return sampleId; + } + + public void setSampleId(Long sampleId) { + this.sampleId = sampleId; } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java index 569b39fac6c..3bbf2effcdf 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java @@ -64,31 +64,30 @@ public ResponseEntity validateNewSampleName(@Reque } /** - * Create a new sample within a project + * Create new samples within a project * - * @param request Details about the sample + * @param requests Details about the samples * @param projectId current project identifier * @param locale current users locale - * @return result of creating the project + * @return result of creating the samples */ @PostMapping("/add-sample") - public ResponseEntity createSampleInProject(@RequestBody CreateSampleRequest request, + public ResponseEntity createSamplesInProject(@RequestBody CreateSampleRequest[] requests, @PathVariable long projectId, Locale locale) { - return uiProjectSampleService.createSample(request, projectId, locale); + return uiProjectSampleService.createSamples(requests, projectId, locale); } /** - * Update a sample within a project + * Update samples within a project * - * @param request Details about the sample - * @param sampleId sample identifier + * @param requests Details about the samples * @param locale current users locale - * @return result of creating the project + * @return result of updating the samples */ - @PatchMapping("/add-sample/{sampleId}") - public ResponseEntity updateSampleInProject(@RequestBody UpdateSampleRequest request, - @PathVariable long sampleId, Locale locale) { - return uiProjectSampleService.updateSample(request, sampleId, locale); + @PatchMapping("/add-sample") + public ResponseEntity updateSamplesInProject(@RequestBody UpdateSampleRequest[] requests, + Locale locale) { + return uiProjectSampleService.updateSamples(requests, locale); } /** diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index 5aa74b27b2d..f51904b671e 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -19,10 +19,7 @@ import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.CreateSampleRequest; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.SampleNameValidationResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.UpdateSampleRequest; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxCreateItemSuccessResponse; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxErrorResponse; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxResponse; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxUpdateItemSuccessResponse; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.*; import ca.corefacility.bioinformatics.irida.ria.web.ajax.projects.dto.ValidateSampleNameModel; import ca.corefacility.bioinformatics.irida.ria.web.ajax.projects.dto.ValidateSampleNamesRequest; import ca.corefacility.bioinformatics.irida.ria.web.ajax.projects.dto.ValidateSampleNamesResponse; @@ -121,16 +118,41 @@ public ResponseEntity validateNewSampleName(String } + /** + * Create new samples in a project + * + * @param requests Each {@link CreateSampleRequest} contains details about the sample to create + * @param projectId Identifier for the current project + * @param locale Users current locale + * @return result of creating the sample + */ + public ResponseEntity createSamples(CreateSampleRequest[] requests, Long projectId, Locale locale) { + Map errors = new HashMap<>(); + for (CreateSampleRequest request : requests) { + try { + createSample(projectId, locale, request); + } catch (Exception e) { + errors.put(request.getName(), e.getMessage()); + } + } + if (errors.isEmpty()) { + return ResponseEntity.ok(new AjaxUpdateItemSuccessResponse( + messageSource.getMessage("server.AddSample.success", null, locale))); + } else { + return ResponseEntity.status(HttpStatus.CONFLICT).body(new AjaxFormErrorResponse(errors)); + } + } + /** * Create a new sample in a project * - * @param request {@link CreateSampleRequest} details about the sample to create + * @param request {@link CreateSampleRequest} contains details about the sample to create * @param projectId Identifier for the current project * @param locale Users current locale * @return result of creating the sample */ @Transactional - public ResponseEntity createSample(CreateSampleRequest request, Long projectId, Locale locale) { + public ResponseEntity createSample(Long projectId, Locale locale, CreateSampleRequest request) { Project project = projectService.read(projectId); try { Sample sample = new Sample(request.getName()); @@ -157,16 +179,40 @@ public ResponseEntity createSample(CreateSampleRequest request, Lo } /** - * Update a sample in a project + * Update samples in a project * - * @param request {@link UpdateSampleRequest} details about the sample to update - * @param sampleId Identifier for the sample + * @param requests Each {@link UpdateSampleRequest} contains details about the sample to update * @param locale Users current locale + * @return result of creating the samples + */ + public ResponseEntity updateSamples(UpdateSampleRequest[] requests, Locale locale) { + Map errors = new HashMap<>(); + for (UpdateSampleRequest request : requests) { + try { + updateSample(request, locale); + } catch (Exception e) { + errors.put(request.getName(), e.getMessage()); + } + } + if (errors.isEmpty()) { + return ResponseEntity.ok(new AjaxUpdateItemSuccessResponse( + messageSource.getMessage("server.AddSample.success", null, locale))); + } else { + return ResponseEntity.status(HttpStatus.CONFLICT).body(new AjaxFormErrorResponse(errors)); + } + } + + /** + * Update a sample in a project + * + * @param request {@link UpdateSampleRequest} contains details about the sample to update + * @param locale Users current locale * @return result of creating the sample */ @Transactional - public ResponseEntity updateSample(UpdateSampleRequest request, Long sampleId, Locale locale) { + public ResponseEntity updateSample(UpdateSampleRequest request, Locale locale) { try { + Long sampleId = request.getSampleId(); Sample sample = sampleService.read(sampleId); sample.setSampleName(request.getName()); if (!Strings.isNullOrEmpty(request.getOrganism())) { diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index 6a6ad1c7df2..7742a83a2b0 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -116,29 +116,24 @@ export async function validateSamples({ return response.data; } -export async function createSample({ +export async function createSamples({ projectId, body, }: { projectId: string; - body: SampleRequest; + body: SampleRequest[]; }) { return await axios.post(`${URL}/${projectId}/samples/add-sample`, body); } -export async function updateSample({ +export async function updateSamples({ projectId, - sampleId, body, }: { projectId: string; - sampleId: number; - body: SampleRequest; + body: SampleRequest[]; }) { - return await axios.patch( - `${URL}/${projectId}/samples/add-sample/${sampleId}`, - body - ); + return await axios.patch(`${URL}/${projectId}/samples/add-sample`, body); } /** diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index 85cd9dabe8e..bebde28c03d 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -5,10 +5,10 @@ import { } from "@reduxjs/toolkit"; import { validateSampleName } from "../../../../apis/metadata/sample-utils"; import { - createSample, + createSamples, FieldUpdate, MetadataItem, - updateSample, + updateSamples, ValidateSampleNameModel, validateSamples, ValidateSamplesResponse, @@ -86,13 +86,15 @@ export const saveMetadata = createAsyncThunk< const sampleId = metadataValidateDetails[index].foundSampleId; if (sampleId) { promises.push( - updateSample({ + updateSamples({ projectId, - sampleId, - body: { - name, - metadata: metadataFields, - }, + // sampleId, + body: [ + { + name, + metadata: metadataFields, + }, + ], }) .then(() => { metadataSaveDetails[index] = { saved: true }; @@ -106,12 +108,14 @@ export const saveMetadata = createAsyncThunk< ); } else { promises.push( - createSample({ + createSamples({ projectId, - body: { - name, - metadata: metadataFields, - }, + body: [ + { + name, + metadata: metadataFields, + }, + ], }) .then(() => { metadataSaveDetails[index] = { saved: true }; diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java index c5aa1d4256d..2c479d17c20 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/unit/web/services/UIProjectSampleServiceTest.java @@ -82,8 +82,8 @@ public void testValidateNewSampleName() { @Test public void testCreateSample() { - CreateSampleRequest request = new CreateSampleRequest(GOOD_NAME, null); - ResponseEntity response = service.createSample(request, PROJECT_1_ID, Locale.ENGLISH); + CreateSampleRequest[] requests = { new CreateSampleRequest(GOOD_NAME, null) }; + ResponseEntity response = service.createSamples(requests, PROJECT_1_ID, Locale.ENGLISH); assertEquals(HttpStatus.OK, response.getStatusCode(), "Sample should be created"); } @@ -92,8 +92,8 @@ public void testCreateSampleWithMetadata() { List metadata = new ArrayList<>(); metadata.add(new FieldUpdate("field1", "value1")); metadata.add(new FieldUpdate("field2", "value2")); - CreateSampleRequest request = new CreateSampleRequest(GOOD_NAME, null, null, metadata); - ResponseEntity response = service.createSample(request, PROJECT_1_ID, Locale.ENGLISH); + CreateSampleRequest[] requests = { new CreateSampleRequest(GOOD_NAME, null, null, metadata) }; + ResponseEntity response = service.createSamples(requests, PROJECT_1_ID, Locale.ENGLISH); assertEquals(HttpStatus.OK, response.getStatusCode(), "Sample should be created"); } @@ -102,8 +102,8 @@ public void testUpdateSampleWithMetadata() { List metadata = new ArrayList<>(); metadata.add(new FieldUpdate("field1", "value1")); metadata.add(new FieldUpdate("field2", "value2")); - UpdateSampleRequest request = new UpdateSampleRequest(GOOD_NAME, null, null, metadata); - ResponseEntity response = service.updateSample(request, SAMPLE_1_ID, Locale.ENGLISH); + UpdateSampleRequest[] requests = { new UpdateSampleRequest(SAMPLE_1_ID, GOOD_NAME, null, null, metadata) }; + ResponseEntity response = service.updateSamples(requests, Locale.ENGLISH); assertEquals(HttpStatus.OK, response.getStatusCode(), "Sample should be updated"); } } From 1492e512fd597bb7e54dfdeecad4db81caa5eb36 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 7 Nov 2022 10:00:02 -0600 Subject: [PATCH 361/655] chore: Initial table setup with spacing --- src/main/webapp/resources/css/app.css | 5 ++ .../resources/js/pages/search/index.tsx | 90 ++++++++++++++++--- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/main/webapp/resources/css/app.css b/src/main/webapp/resources/css/app.css index f4f8f25f2e2..29e37eec654 100644 --- a/src/main/webapp/resources/css/app.css +++ b/src/main/webapp/resources/css/app.css @@ -69,6 +69,11 @@ --primary-red: var(--red-6); --primary-green: var(--green-6); + + --padding-xs: 8px; + --padding-sm: 12px; + --padding-md: 16px; + --padding-lg: 24px; } html, diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index 2b9e1263454..c6fc63e15d5 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -1,5 +1,14 @@ -import { Checkbox, Input, Layout, PageHeader, Space } from "antd"; -import React, { useRef } from "react"; +import { + Checkbox, + Col, + Input, + Layout, + PageHeader, + Row, + Space, + Table, +} from "antd"; +import React, { useState } from "react"; import { render } from "react-dom"; import { createBrowserRouter, @@ -26,19 +35,65 @@ function SearchLayout() { const user = useLoaderData(); console.log(user); const [searchParams, setSearchParams] = useSearchParams(); - const query = useRef(searchParams.get("query")); - console.log(query); + const [query, setQuery] = useState(searchParams.get("query") || ""); return ( - - - - -
    - Project - Samples - {user.admin && Search Global} -
    + + + + + setQuery(e.target.value)} + /> +
    + Project + Samples + {user.admin && Search Global} +
    +     + @@ -47,7 +102,14 @@ function SearchLayout() { const element = document.querySelector("#root"); render( - + , element From c60e3ec8583f98c7cc83313cf5289efcdcd1344a Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 7 Nov 2022 12:46:05 -0600 Subject: [PATCH 362/655] chore: Set up radio buttons for either project or samples search --- .../resources/js/pages/search/index.tsx | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index c6fc63e15d5..4196ad08973 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -4,11 +4,13 @@ import { Input, Layout, PageHeader, + Radio, Row, Space, Table, } from "antd"; -import React, { useState } from "react"; +import type { RadioChangeEvent } from "antd"; +import React, { useEffect, useState } from "react"; import { render } from "react-dom"; import { createBrowserRouter, @@ -27,15 +29,23 @@ const router = createBrowserRouter( path={setBaseUrl("/search")} element={} loader={userLoader} + action={async ({ params, request }) => {}} /> ) ); +type SearchType = "projects" | "samples"; + function SearchLayout() { const user = useLoaderData(); console.log(user); const [searchParams, setSearchParams] = useSearchParams(); const [query, setQuery] = useState(searchParams.get("query") || ""); + const [type, setType] = useState("projects"); + + useEffect(async () => { + await fetch(`/ajax/search/projects`); + }, []); return ( setQuery(e.target.value)} />
    - Project - Samples - {user.admin && Search Global} + + { + setType(e.target.value); + }} + > + Projects + Samples + + {user.admin && Search Global} +
    Date: Mon, 7 Nov 2022 13:09:59 -0600 Subject: [PATCH 363/655] adding table empty --- src/main/resources/i18n/messages.properties | 1 + .../SampleMetadataImportMapHeaders.tsx | 37 ++++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index d14996cfecb..fa698f4f08b 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -1997,6 +1997,7 @@ SampleMetadataImportUploadFile.error={0} failed to upload. SampleMetadataImportMapHeaders.description=Select which column maps to the sample name. SampleMetadataImportMapHeaders.button.back=Upload a new file SampleMetadataImportMapHeaders.button.next=Preview the data +SampleMetadataImportMapHeaders.table.empty=Waiting on the selection of the sample name column. SampleMetadataImportMapHeaders.table.header=Header SampleMetadataImportMapHeaders.table.restriction=Restriction diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 9e12744359d..a62ccdba16f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -1,6 +1,6 @@ import React from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { Button, Radio, Select, Table, Typography } from "antd"; +import { Button, Empty, Radio, Select, Table, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { IconArrowLeft, @@ -112,18 +112,29 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { ))} - {updatedSampleNameColumn && ( -
    row.rowKey} - columns={columns} - dataSource={updatedHeaders.filter( - (updatedHeader) => updatedHeader.name !== updatedSampleNameColumn - )} - pagination={false} - scroll={{ y: 600 }} - /> - )} +
    row.rowKey} + columns={columns} + dataSource={ + updatedSampleNameColumn + ? updatedHeaders.filter( + (updatedHeader) => + updatedHeader.name !== updatedSampleNameColumn + ) + : undefined + } + pagination={false} + scroll={{ y: 600 }} + locale={{ + emptyText: ( + + ), + }} + />
    i18n("SampleMetadataImportMapHeaders.table.title")} className="t-metadata-uploader-headers-table" rowKey={(row) => row.rowKey} columns={columns} From 3ac75f0abfa734bd52cf4594776fc3ba0f20c084 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Mon, 7 Nov 2022 14:18:46 -0600 Subject: [PATCH 365/655] using useMemo for table dataSouce --- .../SampleMetadataImportMapHeaders.tsx | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx index 755911e9a60..5025929a24a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useMemo } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { Button, Empty, Radio, Select, Table, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; @@ -73,6 +73,16 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { } }; + const dataSource = useMemo( + () => + updatedSampleNameColumn + ? updatedHeaders.filter( + (updatedHeader) => updatedHeader.name !== updatedSampleNameColumn + ) + : undefined, + [updatedSampleNameColumn, updatedHeaders] + ); + const columns = [ { title: i18n("SampleMetadataImportMapHeaders.table.header"), @@ -117,14 +127,7 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { className="t-metadata-uploader-headers-table" rowKey={(row) => row.rowKey} columns={columns} - dataSource={ - updatedSampleNameColumn - ? updatedHeaders.filter( - (updatedHeader) => - updatedHeader.name !== updatedSampleNameColumn - ) - : undefined - } + dataSource={dataSource} pagination={false} scroll={{ y: 600 }} locale={{ From 2885cbac79d8743ec9434334f7c933289d5f5fce Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Tue, 8 Nov 2022 10:22:48 -0600 Subject: [PATCH 366/655] chore: Working on ajax request --- .../ria/web/search/SearchController.java | 33 ++--- .../irida/ria/web/search/dto/SearchItem.java | 36 +++++ .../ria/web/search/dto/SearchProject.java | 22 +++ .../ria/web/search/dto/SearchRequest.java | 15 +++ .../ria/web/search/dto/SearchSample.java | 25 ++++ .../resources/js/pages/search/index.tsx | 126 +++++++++++++----- 6 files changed, 207 insertions(+), 50 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchItem.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java index c41de820d2b..1a074036bfd 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java @@ -9,16 +9,19 @@ import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models.DataTablesResponseModel; import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProject; import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProjectSamples; +import ca.corefacility.bioinformatics.irida.ria.web.models.tables.AntTableResponse; +import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchItem; +import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchProject; +import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchRequest; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.google.common.collect.Lists; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Sort; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.*; import java.util.List; import java.util.stream.Collectors; @@ -58,21 +61,19 @@ public String search() { * parameters for a datatables response * @return a {@link DataTablesResponse} to display the search results */ - @RequestMapping("/search/ajax/projects") - @ResponseBody - public DataTablesResponse searchProjects(@RequestParam String query, - @RequestParam(required = false, defaultValue = "false") boolean global, - @DataTablesRequest DataTablesParams params) { + @PostMapping("/ajax/search/projects") + public ResponseEntity> handleSearch(@RequestBody SearchRequest request) { Page page; - if (global) { - page = projectService.findAllProjects(query, params.getCurrentPage(), params.getLength(), params.getSort()); + if (request.isGlobal()) { + page = projectService.findAllProjects((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), request.getSort()); } else { - page = projectService.findProjectsForUser(query, params.getCurrentPage(), params.getLength(), - params.getSort()); + page = projectService.findProjectsForUser((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), request.getSort()); } - List projects = page.getContent().stream().map(this::createDataTablesProject) - .collect(Collectors.toList()); - return new DataTablesResponse(params, page, projects); + AntTableResponse response = new AntTableResponse<>(page.getContent().stream().map(project -> { + Long samples = sampleService.getNumberOfSamplesForProject(project); + return new SearchProject(project, samples); + }).collect(Collectors.toList()), page.getTotalElements()); + return ResponseEntity.ok(response); } /** @@ -84,7 +85,7 @@ public DataTablesResponse searchProjects(@RequestParam String query, * @param params parameters for a datatables response * @return a {@link DataTablesResponse} to display search results */ - @RequestMapping("/search/ajax/samples") + @RequestMapping("/ajax/search/samples") @ResponseBody public DataTablesResponse searchSamples(@RequestParam String query, @RequestParam(required = false, defaultValue = "false") boolean global, diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchItem.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchItem.java new file mode 100644 index 00000000000..ff7e6eb4700 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchItem.java @@ -0,0 +1,36 @@ +package ca.corefacility.bioinformatics.irida.ria.web.search.dto; + +import ca.corefacility.bioinformatics.irida.ria.web.models.tables.AntTableItem; + +import java.util.Date; + +public class SearchItem extends AntTableItem { + private final Long id; + private final String name; + private final Date createdDate; + private final Date modifiedDate; + + public SearchItem(Long id, String name, Date createdDate, Date modifiedDate) { + super(id); + this.id = id; + this.name = name; + this.createdDate = createdDate; + this.modifiedDate = modifiedDate; + } + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public Date getCreatedDate() { + return createdDate; + } + + public Date getModifiedDate() { + return modifiedDate; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java new file mode 100644 index 00000000000..1e76fb5302a --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java @@ -0,0 +1,22 @@ +package ca.corefacility.bioinformatics.irida.ria.web.search.dto; + +import ca.corefacility.bioinformatics.irida.model.project.Project; + +public class SearchProject extends SearchItem { + final String organism; + final Long samples; + + public SearchProject(Project project, Long samples) { + super(project.getId(), project.getName(), project.getCreatedDate(), project.getModifiedDate()); + this.organism = project.getOrganism(); + this.samples = samples; + } + + public String getOrganism() { + return organism; + } + + public Long getSamples() { + return samples; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java new file mode 100644 index 00000000000..15674a286c1 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java @@ -0,0 +1,15 @@ +package ca.corefacility.bioinformatics.irida.ria.web.search.dto; + +import ca.corefacility.bioinformatics.irida.ria.web.models.tables.AntTableRequest; + +public class SearchRequest extends AntTableRequest { + private boolean global; + + public void setGlobal(boolean global) { + this.global = global; + } + + public boolean isGlobal() { + return global; + } +} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java new file mode 100644 index 00000000000..4c18500eaaa --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java @@ -0,0 +1,25 @@ +package ca.corefacility.bioinformatics.irida.ria.web.search.dto; + +import ca.corefacility.bioinformatics.irida.model.sample.Sample; +import ca.corefacility.bioinformatics.irida.ria.web.models.BaseModel; + +import java.util.List; + +public class SearchSample extends BaseModel { + final String organism; + final List projects; + + public SearchSample(Sample sample, List projects) { + super(sample.getId(), sample.getSampleName(), sample.getCreatedDate(), sample.getModifiedDate()); + this.organism = sample.getOrganism(); + this.projects = projects; + } + + public String getOrganism() { + return organism; + } + + public List getProjects() { + return projects; + } +} diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index 4196ad08973..7b456664749 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -1,16 +1,7 @@ -import { - Checkbox, - Col, - Input, - Layout, - PageHeader, - Radio, - Row, - Space, - Table, -} from "antd"; -import type { RadioChangeEvent } from "antd"; -import React, { useEffect, useState } from "react"; +import type { RadioChangeEvent, TablePaginationConfig } from "antd"; +import type { FilterValue, SorterResult } from "antd/es/table/interface"; +import { Checkbox, Input, Layout, PageHeader, Radio, Space, Table } from "antd"; +import React, { useEffect, useMemo, useState } from "react"; import { render } from "react-dom"; import { createBrowserRouter, @@ -22,6 +13,7 @@ import { } from "react-router-dom"; import { setBaseUrl } from "../../utilities/url-utilities"; import userLoader from "./loaders/user-loader"; +import { Sample } from "../../types/irida"; const router = createBrowserRouter( createRoutesFromElements( @@ -35,6 +27,18 @@ const router = createBrowserRouter( ); type SearchType = "projects" | "samples"; +type SampleTableType = Pick< + Sample, + "name" | "organism" | "projects" | "createdDate" | "modifiedDate" +> & { + key: "string"; +}; +interface TableParams { + pagination?: TablePaginationConfig; + sortField?: string; + sortOrder?: string; + filters?: Record; +} function SearchLayout() { const user = useLoaderData(); @@ -43,9 +47,76 @@ function SearchLayout() { const [query, setQuery] = useState(searchParams.get("query") || ""); const [type, setType] = useState("projects"); - useEffect(async () => { - await fetch(`/ajax/search/projects`); - }, []); + const [tableParams, setTableParams] = useState({ + pagination: { + current: 1, + pageSize: 10, + }, + }); + + const fetchProjects = async () => { + const response = await fetch(setBaseUrl(`/ajax/search/projects`), { + method: "POST", + body: JSON.stringify({ + global: true, + pagination: tableParams.pagination, + order: [{ property: "label", direction: `asc` }], + search: [{ property: `label`, value: query, operation: "MATCH_IN" }], + }), + }) + .then((response) => response.json()) + .catch((error) => { + console.error("Error:", error); + }); + console.log(response); + }; + + useEffect(() => { + fetchProjects(); + }, [JSON.stringify(tableParams)]); + + const handleTableChange = ( + pagination: TablePaginationConfig, + filters: Record, + sorter: SorterResult + ) => { + setTableParams({ + pagination, + filters, + ...sorter, + }); + }; + + const columns = useMemo>( + () => [ + { + key: `name`, + dataIndex: "sampleName", + title: "NAME", + }, + { + key: `organism`, + dataIndex: `organism`, + title: "ORGANISM", + }, + { + key: `projects`, + dataIndex: `projects`, + title: `PROJECTS`, + }, + { + key: `createdDate`, + dataIndex: `createdDate`, + title: `CREATED DATE`, + }, + { + key: `modifiedDate`, + dataIndex: `modifiedDate`, + title: `MODIFIED DATE`, + }, + ], + [] + ); return ( setQuery(e.target.value)} + allowClear />
    @@ -90,29 +162,15 @@ function SearchLayout() {
    From e175dac3db370a3428bbf31cf5dd49a2613ee91a Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Wed, 9 Nov 2022 05:05:11 -0600 Subject: [PATCH 367/655] chore: Use react-router's ability to update the query string in the navbar --- .../ria/web/search/SearchController.java | 197 +++++++++--------- .../ria/web/search/dto/SearchSample.java | 3 +- .../js/pages/search/SearchProjectsTable.tsx | 82 ++++++++ .../resources/js/pages/search/index.tsx | 117 ++++------- 4 files changed, 227 insertions(+), 172 deletions(-) create mode 100644 src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java index 1a074036bfd..d54d5c776df 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java @@ -1,18 +1,17 @@ package ca.corefacility.bioinformatics.irida.ria.web.search; +import ca.corefacility.bioinformatics.irida.model.joins.Join; import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesParams; import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesResponse; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config.DataTablesRequest; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models.DataTablesResponseModel; import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProject; import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProjectSamples; import ca.corefacility.bioinformatics.irida.ria.web.models.tables.AntTableResponse; import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchItem; import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchProject; import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchRequest; +import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchSample; import ca.corefacility.bioinformatics.irida.service.ProjectService; import ca.corefacility.bioinformatics.irida.service.sample.SampleService; import com.google.common.collect.Lists; @@ -21,7 +20,10 @@ import org.springframework.data.domain.Sort; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; import java.util.List; import java.util.stream.Collectors; @@ -31,108 +33,105 @@ */ @Controller public class SearchController { - private final ProjectService projectService; - private final SampleService sampleService; + private final ProjectService projectService; + private final SampleService sampleService; - @Autowired - public SearchController(ProjectService projectService, SampleService sampleService) { - this.projectService = projectService; - this.sampleService = sampleService; - } + @Autowired + public SearchController(ProjectService projectService, SampleService sampleService) { + this.projectService = projectService; + this.sampleService = sampleService; + } - /** - * Get the search view - * - * @return name of the search view - */ - @RequestMapping("/search") - public String search() { - return "search/search"; - } + /** + * Get the search view + * + * @return name of the search view + */ + @RequestMapping("/search") + public String search() { + return "search/search"; + } - /** - * Search all projects a user is a member of based on a query string - * - * @param query - * the query string - * @param global - * Whether to perform an admin global search - * @param params - * parameters for a datatables response - * @return a {@link DataTablesResponse} to display the search results - */ - @PostMapping("/ajax/search/projects") - public ResponseEntity> handleSearch(@RequestBody SearchRequest request) { - Page page; - if (request.isGlobal()) { - page = projectService.findAllProjects((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), request.getSort()); - } else { - page = projectService.findProjectsForUser((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), request.getSort()); - } - AntTableResponse response = new AntTableResponse<>(page.getContent().stream().map(project -> { - Long samples = sampleService.getNumberOfSamplesForProject(project); - return new SearchProject(project, samples); - }).collect(Collectors.toList()), page.getTotalElements()); - return ResponseEntity.ok(response); - } + /** + * Search all projects a user is a member of based on a query string + * + * @param query the query string + * @param global Whether to perform an admin global search + * @param params parameters for a datatables response + * @return a {@link DataTablesResponse} to display the search results + */ + @PostMapping("/ajax/search/projects") + public ResponseEntity> handleSearch(@RequestBody SearchRequest request) { + Page page; + if (request.isGlobal()) { + page = projectService.findAllProjects((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), request.getSort()); + } else { + page = projectService.findProjectsForUser((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), request.getSort()); + } + AntTableResponse response = new AntTableResponse<>(page.getContent().stream().map(project -> { + Long samples = sampleService.getNumberOfSamplesForProject(project); + return new SearchProject(project, samples); + }).collect(Collectors.toList()), page.getTotalElements()); + return ResponseEntity.ok(response); + } - /** - * Search all {@link Sample}s in projects for a user based on a query string - * - * @param query the query string - * @param global Whether to perform an admin - * global search - * @param params parameters for a datatables response - * @return a {@link DataTablesResponse} to display search results - */ - @RequestMapping("/ajax/search/samples") - @ResponseBody - public DataTablesResponse searchSamples(@RequestParam String query, - @RequestParam(required = false, defaultValue = "false") boolean global, - @DataTablesRequest DataTablesParams params) { + /** + * Search all {@link Sample}s in projects for a user based on a query string + * + * @param query the query string + * @param global Whether to perform an admin + * global search + * @param params parameters for a datatables response + * @return a {@link DataTablesResponse} to display search results + */ + @RequestMapping("/ajax/search/samples") + @ResponseBody + public ResponseEntity> searchSamples(@RequestBody SearchRequest request) { - Sort originalSort = params.getSort(); - List orders = Lists.newArrayList(); - originalSort.forEach(o -> { - orders.add(new Sort.Order(o.getDirection(), "sample." + o.getProperty())); - }); + Sort originalSort = request.getSort(); + List orders = Lists.newArrayList(); + originalSort.forEach(o -> { + orders.add(new Sort.Order(o.getDirection(), "sample." + o.getProperty())); + }); - Sort newSort = Sort.by(orders); - Page samplePage; - if (global) { - samplePage = sampleService.searchAllSamples(query, params.getCurrentPage(), params.getLength(), newSort); - } else { - samplePage = sampleService.searchSamplesForUser(query, params.getCurrentPage(), params.getLength(), - newSort); - } + Sort newSort = Sort.by(orders); + Page samplePage; + if (request.isGlobal()) { + samplePage = sampleService.searchAllSamples((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), newSort); + } else { + samplePage = sampleService.searchSamplesForUser((String) request.getSearch().get(0).getValue(), request.getPage(), request.getPageSize(), + newSort); + } - List samples = samplePage.getContent().stream().map(this::createDataTablesSample) - .collect(Collectors.toList()); - return new DataTablesResponse(params, samplePage, samples); - } + List samples = samplePage.getContent().stream().map(join -> { + Sample sample = join.getObject(); + List> projects = projectService.getProjectsForSample(sample); + List searchProjects = projects.stream().map(projectSampleJoin -> new SearchProject(projectSampleJoin.getSubject(), 0L)).collect(Collectors.toList()); + return new SearchSample(sample, searchProjects); + }).collect(Collectors.toList()); + AntTableResponse response = new AntTableResponse<>(samples, samplePage.getTotalElements()); + return ResponseEntity.ok(response); + } - /** - * Extract the details of the a {@link Project} into a {@link DTProject} - * which is consumable by the UI - * - * @param project - * {@link Project} - * - * @return {@link DTProject} - */ - private DTProject createDataTablesProject(Project project) { - return new DTProject(project, sampleService.getNumberOfSamplesForProject(project)); - } + /** + * Extract the details of the a {@link Project} into a {@link DTProject} + * which is consumable by the UI + * + * @param project {@link Project} + * @return {@link DTProject} + */ + private DTProject createDataTablesProject(Project project) { + return new DTProject(project, sampleService.getNumberOfSamplesForProject(project)); + } - /** - * Extract the details of a {@link ProjectSampleJoin} into a - * {@link DTProjectSamples} - * - * @param join - * the {@link ProjectSampleJoin} - * @return the created {@link DTProjectSamples} - */ - private DTProjectSamples createDataTablesSample(ProjectSampleJoin join) { - return new DTProjectSamples(join, Lists.newArrayList(), null); - } + /** + * Extract the details of a {@link ProjectSampleJoin} into a + * {@link DTProjectSamples} + * + * @param join the {@link ProjectSampleJoin} + * @return the created {@link DTProjectSamples} + */ + private DTProjectSamples createDataTablesSample(ProjectSampleJoin join) { + return new DTProjectSamples(join, Lists.newArrayList(), null); + } } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java index 4c18500eaaa..c393b729cae 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java @@ -1,11 +1,10 @@ package ca.corefacility.bioinformatics.irida.ria.web.search.dto; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.ria.web.models.BaseModel; import java.util.List; -public class SearchSample extends BaseModel { +public class SearchSample extends SearchItem { final String organism; final List projects; diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx new file mode 100644 index 00000000000..dcdfffdf300 --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -0,0 +1,82 @@ +import React, { useMemo } from "react"; +import type { ColumnType } from "antd/es/list"; +import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; +import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; +import { Table, TablePaginationConfig } from "antd"; +import axios from "axios"; +import { setBaseUrl } from "../../utilities/url-utilities"; +import { FilterValue, SorterResult } from "antd/es/table/interface"; +import { TableParams } from "./index"; + +type SearchItem = { + id: number; + name: string; + createdDate: number; + modifiedDate: number; + organism: string; +}; + +type SearchProject = SearchItem & { + samples: SearchSample[]; +}; + +type SearchSample = SearchItem & { + projects: SearchProject[]; +}; + +type SearchProjectsTableParams = { + projects: SearchProject[]; + total: number; + handleTableChange: ( + pagination: TablePaginationConfig, + filters: Record, + sorter: SorterResult + ) => TableParams; +}; +export default function SearchProjectsTable({ + projects, + total, + handleTableChange, +}: SearchProjectsTableParams) { + const columns = useMemo[]>( + () => [ + { + key: `name`, + dataIndex: "name", + title: "NAME", + }, + { + key: `organism`, + dataIndex: `organism`, + title: "ORGANISM", + }, + { + key: `samples`, + dataIndex: `samples`, + title: `SAMPLES`, + }, + { + key: `createdDate`, + dataIndex: `createdDate`, + title: `CREATED DATE`, + render: (text) => formatInternationalizedDateTime(text), + }, + { + key: `modifiedDate`, + dataIndex: `modifiedDate`, + title: `MODIFIED DATE`, + render: (text) => formatInternationalizedDateTime(text), + }, + ], + [] + ); + + return ( +
    + ); +} diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index 7b456664749..aad1ae26269 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -14,6 +14,11 @@ import { import { setBaseUrl } from "../../utilities/url-utilities"; import userLoader from "./loaders/user-loader"; import { Sample } from "../../types/irida"; +import axios from "axios"; +import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; +import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; +import { ColumnType } from "antd/es/list"; +import SearchProjectsTable from "./SearchProjectsTable"; const router = createBrowserRouter( createRoutesFromElements( @@ -33,7 +38,7 @@ type SampleTableType = Pick< > & { key: "string"; }; -interface TableParams { +export interface TableParams { pagination?: TablePaginationConfig; sortField?: string; sortOrder?: string; @@ -41,11 +46,14 @@ interface TableParams { } function SearchLayout() { + k; const user = useLoaderData(); - console.log(user); const [searchParams, setSearchParams] = useSearchParams(); - const [query, setQuery] = useState(searchParams.get("query") || ""); const [type, setType] = useState("projects"); + const [global, setGlobal] = useState(false); + + const [projects, setProjects] = useState(); + const [projectsTotal, setProjectsTotal] = useState(); const [tableParams, setTableParams] = useState({ pagination: { @@ -54,27 +62,6 @@ function SearchLayout() { }, }); - const fetchProjects = async () => { - const response = await fetch(setBaseUrl(`/ajax/search/projects`), { - method: "POST", - body: JSON.stringify({ - global: true, - pagination: tableParams.pagination, - order: [{ property: "label", direction: `asc` }], - search: [{ property: `label`, value: query, operation: "MATCH_IN" }], - }), - }) - .then((response) => response.json()) - .catch((error) => { - console.error("Error:", error); - }); - console.log(response); - }; - - useEffect(() => { - fetchProjects(); - }, [JSON.stringify(tableParams)]); - const handleTableChange = ( pagination: TablePaginationConfig, filters: Record, @@ -86,43 +73,36 @@ function SearchLayout() { ...sorter, }); }; + const fetchProjects = async () => { + return axios.post(setBaseUrl(`/ajax/search/projects`), { + global, + pagination: tableParams.pagination, + order: [{ property: "name", direction: `asc` }], + search: [ + { + property: `name`, + value: searchParams.get("query"), + operation: "MATCH_IN", + }, + ], + }); + }; - const columns = useMemo>( - () => [ - { - key: `name`, - dataIndex: "sampleName", - title: "NAME", - }, - { - key: `organism`, - dataIndex: `organism`, - title: "ORGANISM", - }, - { - key: `projects`, - dataIndex: `projects`, - title: `PROJECTS`, - }, - { - key: `createdDate`, - dataIndex: `createdDate`, - title: `CREATED DATE`, - }, - { - key: `modifiedDate`, - dataIndex: `modifiedDate`, - title: `MODIFIED DATE`, - }, - ], - [] - ); + useEffect(() => { + fetchProjects() + .then(({ data }) => { + setProjects(data.content); + setProjectsTotal(data.total); + }) + .catch((error) => { + console.error("Error:", error); + }); + }, [fetchProjects, global, searchParams]); return ( setQuery(e.target.value)} + defaultValue={searchParams.get("query") || ""} + onChange={(e) => setSearchParams({ query: e.target.value })} allowClear />
    @@ -155,22 +135,18 @@ function SearchLayout() { Projects Samples - {user.admin && Search Global} + {user.admin && ( + setGlobal(e.target.checked)}> + Search Global + + )}
    -
    @@ -183,7 +159,6 @@ render( Date: Wed, 9 Nov 2022 10:14:26 -0600 Subject: [PATCH 368/655] sending requests in chunks --- .../services/importReducer.ts | 141 ++++++++++-------- .../resources/js/utilities/array-utilities.ts | 18 +++ 2 files changed, 93 insertions(+), 66 deletions(-) create mode 100644 src/main/webapp/resources/js/utilities/array-utilities.ts diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index bebde28c03d..415083a28be 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -6,7 +6,6 @@ import { import { validateSampleName } from "../../../../apis/metadata/sample-utils"; import { createSamples, - FieldUpdate, MetadataItem, updateSamples, ValidateSampleNameModel, @@ -14,6 +13,7 @@ import { ValidateSamplesResponse, } from "../../../../apis/projects/samples"; import { ImportDispatch, ImportState } from "../store"; +import { chunkArray } from "../../../../utilities/array-utilities"; interface MetadataValidateDetailsItem { isSampleNameValid: boolean; @@ -67,76 +67,85 @@ export const saveMetadata = createAsyncThunk< state.importReducer; const metadataSaveDetails: Record = {}; + const selectedSampleList = metadata.filter((metadataItem) => { + const index: string = metadataItem.rowKey; + return ( + selectedMetadataKeys.includes(index) && + metadataSaveDetails[index]?.saved !== true + ); + }); const chunkSize = 100; - for (let i = 0; i < metadata.length; i = i + chunkSize) { - const promises: Promise[] = []; - for (let j = i; j < i + chunkSize && j < metadata.length; j++) { - const metadataItem: MetadataItem = metadata[j]; - const index: string = metadataItem.rowKey; - if ( - selectedMetadataKeys.includes(index) && - metadataSaveDetails[index]?.saved !== true - ) { - const name: string = metadataItem[sampleNameColumn]; - const metadataFields: FieldUpdate[] = Object.entries(metadataItem) - .filter( - ([key]) => headers.includes(key) && key !== sampleNameColumn - ) - .map(([key, value]) => ({ field: key, value })); - const sampleId = metadataValidateDetails[index].foundSampleId; - if (sampleId) { - promises.push( - updateSamples({ - projectId, - // sampleId, - body: [ - { - name, - metadata: metadataFields, - }, - ], - }) - .then(() => { - metadataSaveDetails[index] = { saved: true }; - }) - .catch((error) => { - metadataSaveDetails[index] = { - saved: false, - error: error.response.data.error, - }; - }) - ); - } else { - promises.push( - createSamples({ - projectId, - body: [ - { - name, - metadata: metadataFields, - }, - ], - }) - .then(() => { - metadataSaveDetails[index] = { saved: true }; - }) - .catch((error) => { - metadataSaveDetails[index] = { - saved: false, - error: error.response.data.error, - }; - }) - ); - } - } + + const updateSampleList = selectedSampleList + .filter((metadataItem) => { + const index = metadataItem.rowKey; + const sampleId = metadataValidateDetails[index].foundSampleId; + return sampleId; + }) + .map((metadataItem) => { + const index = metadataItem.rowKey; + const name = metadataItem[sampleNameColumn]; + const sampleId = metadataValidateDetails[index].foundSampleId; + const metadataFields = Object.entries(metadataItem) + .filter(([key]) => headers.includes(key) && key !== sampleNameColumn) + .map(([key, value]) => ({ field: key, value })); + return { name, sampleId, metadata: metadataFields }; + }); + + if (updateSampleList.length > 0) { + const chunkedUpdateSampleList = chunkArray(updateSampleList, chunkSize); + for (const chunk of chunkedUpdateSampleList) { + await updateSamples({ + projectId, + body: chunk, + }) + .then((response) => { + // metadataSaveDetails[index] = { saved: true }; + }) + .catch((error) => { + // metadataSaveDetails[index] = { + // saved: false, + // error: error.response.data.error, + // }; + }); } - await Promise.all(promises).then(() => { - dispatch( - setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) - ); + } + + const createSampleList = selectedSampleList + .filter((metadataItem) => { + const index = metadataItem.rowKey; + const sampleId = metadataValidateDetails[index].foundSampleId; + return !sampleId; + }) + .map((metadataItem) => { + const name = metadataItem[sampleNameColumn]; + const metadataFields = Object.entries(metadataItem) + .filter(([key]) => headers.includes(key) && key !== sampleNameColumn) + .map(([key, value]) => ({ field: key, value })); + return { name, metadata: metadataFields }; }); + + if (createSampleList.length > 0) { + const chunkedCreateSampleList = chunkArray(createSampleList, chunkSize); + for (const chunk of chunkedCreateSampleList) { + await createSamples({ + projectId, + body: chunk, + }) + .then((response) => { + // metadataSaveDetails[index] = { saved: true }; + }) + .catch((error) => { + // metadataSaveDetails[index] = { + // saved: false, + // error: error.response.data.error, + // }; + }); + } } + dispatch(setMetadataSaveDetails(Object.assign({}, metadataSaveDetails))); + return { metadataSaveDetails }; } ); diff --git a/src/main/webapp/resources/js/utilities/array-utilities.ts b/src/main/webapp/resources/js/utilities/array-utilities.ts new file mode 100644 index 00000000000..7cf06ec9492 --- /dev/null +++ b/src/main/webapp/resources/js/utilities/array-utilities.ts @@ -0,0 +1,18 @@ +/** + * Splits an array into multiple, smaller arrays of the given size + * + * @param items the array to be chunked + * @param size the size of the chunk + * + * @returns a chunked array + */ +export function chunkArray(items: any[], size: number) { + const chunks = []; + items = [].concat(...items); + + while (items.length) { + chunks.push(items.splice(0, size)); + } + + return chunks; +} From 4f71dfe3df0053d6936c3ec9fd22f3c5eeb4a357 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Wed, 9 Nov 2022 13:40:53 -0600 Subject: [PATCH 369/655] chore: Updated project tags to use a common component ProjectTag --- .../webapp/resources/js/apis/users/user.ts | 10 +- .../samples/components/SamplesTable.jsx | 24 +- .../resources/js/pages/search/ProjectTag.tsx | 39 +++ .../js/pages/search/SearchLayout.tsx | 279 ++++++++++++++++++ .../js/pages/search/SearchProjectsTable.tsx | 43 +-- .../js/pages/search/SearchSamplesTable.tsx | 74 +++++ .../resources/js/pages/search/index.tsx | 139 +-------- .../js/pages/search/loaders/user-loader.ts | 6 +- .../resources/js/utilities/date-utilities.ts | 2 +- 9 files changed, 426 insertions(+), 190 deletions(-) create mode 100644 src/main/webapp/resources/js/pages/search/ProjectTag.tsx create mode 100644 src/main/webapp/resources/js/pages/search/SearchLayout.tsx create mode 100644 src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx diff --git a/src/main/webapp/resources/js/apis/users/user.ts b/src/main/webapp/resources/js/apis/users/user.ts index b863d4a7246..6d9c4940e44 100644 --- a/src/main/webapp/resources/js/apis/users/user.ts +++ b/src/main/webapp/resources/js/apis/users/user.ts @@ -5,11 +5,19 @@ import { setBaseUrl } from "../../utilities/url-utilities"; * @file API methods for the currently logged in user */ +export type CurrentUser = { + admin: boolean; + firstName: string; + identifier: number; + lastName: string; + username: string; +}; + /** * Get details about the currently logged in user * @returns {Promise} */ -export async function fetchCurrentUserDetails() { +export async function fetchCurrentUserDetails(): Promise { try { const { data } = await axios.get(setBaseUrl(`/ajax/users/current`)); return Promise.resolve(data); diff --git a/src/main/webapp/resources/js/pages/projects/samples/components/SamplesTable.jsx b/src/main/webapp/resources/js/pages/projects/samples/components/SamplesTable.jsx index f5e6e46f8db..9f6393c62b5 100644 --- a/src/main/webapp/resources/js/pages/projects/samples/components/SamplesTable.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples/components/SamplesTable.jsx @@ -23,6 +23,7 @@ import { blue6 } from "../../../../styles/colors"; import { generateColourForItem } from "../../../../utilities/colour-utilities"; import { getPaginationOptions } from "../../../../utilities/antdesign-table-utilities"; import { SampleDetailViewer } from "../../../../components/samples/SampleDetailViewer"; +import ProjectTag from "../../../search/ProjectTag"; const { RangePicker } = DatePicker; @@ -116,8 +117,6 @@ export function SamplesTable() { confirm({ closeDropdown: false }); }; - const projectColours = {}; - const getColumnSearchProps = ( dataIndex, filterName = "", @@ -292,26 +291,7 @@ export function SamplesTable() { sorter: true, key: "associated", render: (name, row) => { - if (!(row.project.id in projectColours)) { - projectColours[row.project.id] = generateColourForItem({ - id: row.project.id, - label: name, - }); - } - const colour = projectColours[row.project.id]; - return ( - - - {name} - - - ); + return ; }, filters: associatedProjects, }, diff --git a/src/main/webapp/resources/js/pages/search/ProjectTag.tsx b/src/main/webapp/resources/js/pages/search/ProjectTag.tsx new file mode 100644 index 00000000000..7ac6ff204e7 --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/ProjectTag.tsx @@ -0,0 +1,39 @@ +import React from "react"; +import { generateColourForItem } from "../../utilities/colour-utilities"; +import { Tag } from "antd"; +import { setBaseUrl } from "../../utilities/url-utilities"; + +const colourMap = new Map(); +export default function ProjectTag({ + project, +}: { + project: { id: number; name: string }; +}) { + if (!(project.id in colourMap)) { + colourMap.set( + project.id, + generateColourForItem({ + id: project.id, + label: project.name, + }) + ); + } + + const colour = colourMap.get(project.id); + if (!colour) throw new Error("THIS IS JUST A TYPE CHECK COLOUR SHOULD EXIST"); + + return ( + + + {project.name} + + + ); +} diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx new file mode 100644 index 00000000000..8106d038460 --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -0,0 +1,279 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { useLoaderData, useSearchParams } from "react-router-dom"; +import SearchProjectsTable from "./SearchProjectsTable"; +import SearchSamplesTable from "./SearchSamplesTable"; +import { + Badge, + Input, + Layout, + Menu, + PageHeader, + Select, + Space, + TablePaginationConfig, + Typography, +} from "antd"; +import { FilterValue, SorterResult } from "antd/es/table/interface"; +import axios from "axios"; +import { setBaseUrl } from "../../utilities/url-utilities"; +import { Sample } from "../../types/irida"; +import { CurrentUser } from "../../apis/users/user"; + +type SearchType = "projects" | "samples"; +type SearchItem = { + id: number; + name: string; + createdDate: number; + modifiedDate: number; + organism: string; +}; + +export type SearchProject = SearchItem & { + samples: SearchSample[]; +}; + +export type SearchSample = SearchItem & { + projects: SearchProject[]; +}; +export type SampleTableType = Pick< + Sample, + "name" | "organism" | "projects" | "createdDate" | "modifiedDate" +> & { + key: "string"; +}; +export interface TableParams { + pagination?: TablePaginationConfig; + sortField?: string; + sortOrder?: string; + filters?: Record; +} + +const initial_table_params = JSON.stringify({ + pagination: { + current: 1, + pageSize: 10, + }, +}); + +export default function SearchLayout() { + const user = useLoaderData() as CurrentUser; + const [searchParams, setSearchParams] = useSearchParams(); + const [type, setType] = useState("projects"); + const [global, setGlobal] = useState(user.admin); + + const [projects, setProjects] = useState<{ + content: SearchProject[]; + total: number; + }>(); + const [projectsTableParams, setProjectsTableParams] = useState( + JSON.parse(initial_table_params) + ); + + const [samples, setSamples] = useState<{ + content: SearchSample[]; + total: number; + }>(); + const [samplesTableParams, setSamplesTableParams] = useState( + JSON.parse(initial_table_params) + ); + + const handleProjectsTableChange = ( + pagination: TablePaginationConfig, + filters: Record, + sorter: SorterResult + ) => { + setProjectsTableParams({ + pagination, + filters, + ...sorter, + }); + }; + + const handleSamplesTableChange = ( + pagination: TablePaginationConfig, + filters: Record, + sorter: SorterResult + ) => { + setSamplesTableParams({ + pagination, + filters, + ...sorter, + }); + }; + + const fetchData = useCallback(() => { + const fetchSamples = async () => + axios.post(setBaseUrl(`/ajax/search/samples`), { + global, + pagination: samplesTableParams.pagination, + order: [{ property: "sampleName", direction: `asc` }], + search: [ + { + property: `name`, + value: searchParams.get("query"), + operation: "MATCH_IN", + }, + ], + }); + + const fetchProjects = async () => + axios.post(setBaseUrl(`/ajax/search/projects`), { + global, + pagination: projectsTableParams.pagination, + order: [{ property: "name", direction: `asc` }], + search: [ + { + property: `name`, + value: searchParams.get("query"), + operation: "MATCH_IN", + }, + ], + }); + + const promises = []; + promises.push(fetchProjects()); + promises.push(fetchSamples()); + Promise.all(promises).then( + ([{ data: projectsData }, { data: samplesData }]) => { + setProjects(projectsData); + setSamples(samplesData); + } + ); + }, [ + global, + projectsTableParams.pagination, + samplesTableParams.pagination, + searchParams, + ]); + + useEffect(() => { + fetchData(); + }, [fetchData, global, searchParams]); + + const menuItems = useMemo( + () => [ + { + label: ( +
    + PROJECTS + +
    + ), + key: "projects", + }, + { + label: ( +
    + SAMPLES + +
    + ), + key: "samples", + }, + ], + [projects?.total, samples?.total] + ); + + const searchPrefix = ( + + ); + + return ( + + + + + + What are you searching for? + + setSearchParams({ query: e.target.value })} + allowClear + /> + + + + setType(e.key as SearchType)} + > + {menuItems.map((item) => ( + {item.label} + ))} + + + + {type === "projects" ? ( + + ) : ( + + )} + + + + + + ); +} diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index dcdfffdf300..51379a4f2c4 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -3,30 +3,17 @@ import type { ColumnType } from "antd/es/list"; import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; import { Table, TablePaginationConfig } from "antd"; -import axios from "axios"; -import { setBaseUrl } from "../../utilities/url-utilities"; import { FilterValue, SorterResult } from "antd/es/table/interface"; -import { TableParams } from "./index"; - -type SearchItem = { - id: number; - name: string; - createdDate: number; - modifiedDate: number; - organism: string; -}; - -type SearchProject = SearchItem & { - samples: SearchSample[]; -}; - -type SearchSample = SearchItem & { - projects: SearchProject[]; -}; +import { SampleTableType, SearchProject, TableParams } from "./SearchLayout"; +import ProjectTag from "./ProjectTag"; type SearchProjectsTableParams = { - projects: SearchProject[]; - total: number; + projects: + | { + content: SearchProject[]; + total: number; + } + | undefined; handleTableChange: ( pagination: TablePaginationConfig, filters: Record, @@ -35,15 +22,15 @@ type SearchProjectsTableParams = { }; export default function SearchProjectsTable({ projects, - total, handleTableChange, }: SearchProjectsTableParams) { - const columns = useMemo[]>( + const columns = useMemo( () => [ { key: `name`, dataIndex: "name", title: "NAME", + render: (_, project) => , }, { key: `organism`, @@ -59,13 +46,13 @@ export default function SearchProjectsTable({ key: `createdDate`, dataIndex: `createdDate`, title: `CREATED DATE`, - render: (text) => formatInternationalizedDateTime(text), + render: (text: string) => formatInternationalizedDateTime(text), }, { key: `modifiedDate`, dataIndex: `modifiedDate`, title: `MODIFIED DATE`, - render: (text) => formatInternationalizedDateTime(text), + render: (text: string) => formatInternationalizedDateTime(text), }, ], [] @@ -73,9 +60,11 @@ export default function SearchProjectsTable({ return (
    ); diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx new file mode 100644 index 00000000000..132bcd87bc7 --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -0,0 +1,74 @@ +import React, { useMemo } from "react"; +import type { ColumnType } from "antd/es/list"; +import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; +import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; +import { Table, TablePaginationConfig, Tag } from "antd"; +import { FilterValue, SorterResult } from "antd/es/table/interface"; +import { TableParams } from "./index"; +import { SampleTableType, SearchSample } from "./SearchLayout"; +import ProjectTag from "./ProjectTag"; + +type SearchSamplesTableParams = { + samples: + | { + content: SearchSample[]; + total: number; + } + | undefined; + handleTableChange: ( + pagination: TablePaginationConfig, + filters: Record, + sorter: SorterResult + ) => TableParams; +}; +export default function SearchSamplesTable({ + samples, + handleTableChange, +}: SearchSamplesTableParams) { + const columns = useMemo( + () => [ + { + key: `name`, + dataIndex: "name", + title: "NAME", + }, + { + key: `organism`, + dataIndex: `organism`, + title: "ORGANISM", + }, + { + key: `projects`, + dataIndex: `projects`, + title: `PROJECTS`, + render: (projects) => { + return projects.map((project) => ); + }, + }, + { + key: `createdDate`, + dataIndex: `createdDate`, + title: `CREATED DATE`, + render: (text: string) => formatInternationalizedDateTime(text), + }, + { + key: `modifiedDate`, + dataIndex: `modifiedDate`, + title: `MODIFIED DATE`, + render: (text: string) => formatInternationalizedDateTime(text), + }, + ], + [] + ); + + return ( +
    + ); +} diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index aad1ae26269..02d888344e9 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -1,24 +1,15 @@ -import type { RadioChangeEvent, TablePaginationConfig } from "antd"; -import type { FilterValue, SorterResult } from "antd/es/table/interface"; -import { Checkbox, Input, Layout, PageHeader, Radio, Space, Table } from "antd"; -import React, { useEffect, useMemo, useState } from "react"; +import { Layout } from "antd"; +import React from "react"; import { render } from "react-dom"; import { createBrowserRouter, createRoutesFromElements, Route, RouterProvider, - useLoaderData, - useSearchParams, } from "react-router-dom"; import { setBaseUrl } from "../../utilities/url-utilities"; import userLoader from "./loaders/user-loader"; -import { Sample } from "../../types/irida"; -import axios from "axios"; -import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; -import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; -import { ColumnType } from "antd/es/list"; -import SearchProjectsTable from "./SearchProjectsTable"; +import SearchLayout from "./SearchLayout"; const router = createBrowserRouter( createRoutesFromElements( @@ -26,134 +17,10 @@ const router = createBrowserRouter( path={setBaseUrl("/search")} element={} loader={userLoader} - action={async ({ params, request }) => {}} /> ) ); -type SearchType = "projects" | "samples"; -type SampleTableType = Pick< - Sample, - "name" | "organism" | "projects" | "createdDate" | "modifiedDate" -> & { - key: "string"; -}; -export interface TableParams { - pagination?: TablePaginationConfig; - sortField?: string; - sortOrder?: string; - filters?: Record; -} - -function SearchLayout() { - k; - const user = useLoaderData(); - const [searchParams, setSearchParams] = useSearchParams(); - const [type, setType] = useState("projects"); - const [global, setGlobal] = useState(false); - - const [projects, setProjects] = useState(); - const [projectsTotal, setProjectsTotal] = useState(); - - const [tableParams, setTableParams] = useState({ - pagination: { - current: 1, - pageSize: 10, - }, - }); - - const handleTableChange = ( - pagination: TablePaginationConfig, - filters: Record, - sorter: SorterResult - ) => { - setTableParams({ - pagination, - filters, - ...sorter, - }); - }; - const fetchProjects = async () => { - return axios.post(setBaseUrl(`/ajax/search/projects`), { - global, - pagination: tableParams.pagination, - order: [{ property: "name", direction: `asc` }], - search: [ - { - property: `name`, - value: searchParams.get("query"), - operation: "MATCH_IN", - }, - ], - }); - }; - - useEffect(() => { - fetchProjects() - .then(({ data }) => { - setProjects(data.content); - setProjectsTotal(data.total); - }) - .catch((error) => { - console.error("Error:", error); - }); - }, [fetchProjects, global, searchParams]); - - return ( - - - - - setSearchParams({ query: e.target.value })} - allowClear - /> -
    - - { - setType(e.target.value); - }} - > - Projects - Samples - - {user.admin && ( - setGlobal(e.target.checked)}> - Search Global - - )} - -
    -     - -     -     -     - ); -} - const element = document.querySelector("#root"); render( { + return await fetchCurrentUserDetails(); } diff --git a/src/main/webapp/resources/js/utilities/date-utilities.ts b/src/main/webapp/resources/js/utilities/date-utilities.ts index 450a50d3428..7d83647e9bc 100644 --- a/src/main/webapp/resources/js/utilities/date-utilities.ts +++ b/src/main/webapp/resources/js/utilities/date-utilities.ts @@ -3,7 +3,7 @@ import relativeTime from "dayjs/plugin/relativeTime"; import duration from "dayjs/plugin/duration"; export function formatInternationalizedDateTime( - date: Date, + date: number | string, options = { hour: "numeric", minute: "numeric", From 6aa9d6c89cec2f1694bd57d98120ed1ddc99d29c Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 9 Nov 2022 16:02:19 -0600 Subject: [PATCH 370/655] fixing error handling --- .../web/services/UIProjectSampleService.java | 98 +++++++++---------- .../SampleMetadataImportComplete.tsx | 16 +-- .../components/SampleMetadataImportReview.tsx | 30 +++--- .../services/importReducer.ts | 68 +++++++------ 4 files changed, 115 insertions(+), 97 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index f51904b671e..950840dfd05 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -3,6 +3,8 @@ import java.util.*; import java.util.stream.Collectors; +import javax.validation.ConstraintViolationException; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.http.HttpStatus; @@ -19,7 +21,9 @@ import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.CreateSampleRequest; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.SampleNameValidationResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.UpdateSampleRequest; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.*; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxFormErrorResponse; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxResponse; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxUpdateItemSuccessResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.projects.dto.ValidateSampleNameModel; import ca.corefacility.bioinformatics.irida.ria.web.ajax.projects.dto.ValidateSampleNamesRequest; import ca.corefacility.bioinformatics.irida.ria.web.ajax.projects.dto.ValidateSampleNamesResponse; @@ -130,7 +134,7 @@ public ResponseEntity createSamples(CreateSampleRequest[] requests Map errors = new HashMap<>(); for (CreateSampleRequest request : requests) { try { - createSample(projectId, locale, request); + createSample(projectId, request); } catch (Exception e) { errors.put(request.getName(), e.getMessage()); } @@ -148,34 +152,29 @@ public ResponseEntity createSamples(CreateSampleRequest[] requests * * @param request {@link CreateSampleRequest} contains details about the sample to create * @param projectId Identifier for the current project - * @param locale Users current locale * @return result of creating the sample + * @throws EntityNotFoundException if the identifier does not exist in the database */ @Transactional - public ResponseEntity createSample(Long projectId, Locale locale, CreateSampleRequest request) { + public Long createSample(Long projectId, CreateSampleRequest request) throws EntityNotFoundException { Project project = projectService.read(projectId); - try { - Sample sample = new Sample(request.getName()); - if (!Strings.isNullOrEmpty(request.getOrganism())) { - sample.setOrganism(request.getOrganism()); - } - if (!Strings.isNullOrEmpty(request.getDescription())) { - sample.setDescription(request.getDescription()); - } - Join join = projectService.addSampleToProjectWithoutEvent(project, sample, true); - if (request.getMetadata() != null) { - Set metadataEntrySet = request.getMetadata().stream().map(entry -> { - MetadataTemplateField field = metadataTemplateService.saveMetadataField( - new MetadataTemplateField(entry.getField(), "text")); - return new MetadataEntry(entry.getValue(), "text", field); - }).collect(Collectors.toSet()); - sampleService.mergeSampleMetadata(sample, metadataEntrySet); - } - return ResponseEntity.ok(new AjaxCreateItemSuccessResponse(join.getObject().getId())); - } catch (EntityNotFoundException e) { - return ResponseEntity.ok(new AjaxErrorResponse( - messageSource.getMessage("server.AddSample.error.exists", new Object[] {}, locale))); + Sample sample = new Sample(request.getName()); + if (!Strings.isNullOrEmpty(request.getOrganism())) { + sample.setOrganism(request.getOrganism()); + } + if (!Strings.isNullOrEmpty(request.getDescription())) { + sample.setDescription(request.getDescription()); + } + Join join = projectService.addSampleToProjectWithoutEvent(project, sample, true); + if (request.getMetadata() != null) { + Set metadataEntrySet = request.getMetadata().stream().map(entry -> { + MetadataTemplateField field = metadataTemplateService.saveMetadataField( + new MetadataTemplateField(entry.getField(), "text")); + return new MetadataEntry(entry.getValue(), "text", field); + }).collect(Collectors.toSet()); + sampleService.mergeSampleMetadata(sample, metadataEntrySet); } + return join.getObject().getId(); } /** @@ -189,7 +188,7 @@ public ResponseEntity updateSamples(UpdateSampleRequest[] requests Map errors = new HashMap<>(); for (UpdateSampleRequest request : requests) { try { - updateSample(request, locale); + updateSample(request); } catch (Exception e) { errors.put(request.getName(), e.getMessage()); } @@ -206,34 +205,31 @@ public ResponseEntity updateSamples(UpdateSampleRequest[] requests * Update a sample in a project * * @param request {@link UpdateSampleRequest} contains details about the sample to update - * @param locale Users current locale * @return result of creating the sample + * @throws EntityNotFoundException if the identifier does not exist in the database + * @throws ConstraintViolationException if the entity being updated contains constraint violations */ @Transactional - public ResponseEntity updateSample(UpdateSampleRequest request, Locale locale) { - try { - Long sampleId = request.getSampleId(); - Sample sample = sampleService.read(sampleId); - sample.setSampleName(request.getName()); - if (!Strings.isNullOrEmpty(request.getOrganism())) { - sample.setOrganism(request.getOrganism()); - } - if (request.getDescription() != null) { - sample.setDescription(request.getDescription()); - } - if (request.getMetadata() != null) { - Set metadataEntrySet = request.getMetadata().stream().map(entry -> { - MetadataTemplateField field = metadataTemplateService.saveMetadataField( - new MetadataTemplateField(entry.getField(), "text")); - return new MetadataEntry(entry.getValue(), "text", field); - }).collect(Collectors.toSet()); - sampleService.updateSampleMetadata(sample, metadataEntrySet); - } - sampleService.update(sample); - return ResponseEntity.ok(new AjaxUpdateItemSuccessResponse( - messageSource.getMessage("server.AddSample.success", null, locale))); - } catch (Exception e) { - return ResponseEntity.status(HttpStatus.CONFLICT).body(new AjaxErrorResponse(e.getMessage())); + public Sample updateSample(UpdateSampleRequest request) + throws EntityNotFoundException, ConstraintViolationException { + Long sampleId = request.getSampleId(); + Sample sample = sampleService.read(sampleId); + sample.setSampleName(request.getName()); + if (!Strings.isNullOrEmpty(request.getOrganism())) { + sample.setOrganism(request.getOrganism()); + } + if (request.getDescription() != null) { + sample.setDescription(request.getDescription()); } + if (request.getMetadata() != null) { + Set metadataEntrySet = request.getMetadata().stream().map(entry -> { + MetadataTemplateField field = metadataTemplateService.saveMetadataField( + new MetadataTemplateField(entry.getField(), "text")); + return new MetadataEntry(entry.getValue(), "text", field); + }).collect(Collectors.toSet()); + sampleService.updateSampleMetadata(sample, metadataEntrySet); + } + return sampleService.update(sample); + } } diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx index 6a05b9bba2c..e5f1760631f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportComplete.tsx @@ -14,19 +14,23 @@ import { NavigateFunction } from "react-router/dist/lib/hooks"; * @constructor */ export function SampleMetadataImportComplete(): JSX.Element { - const { metadata, metadataValidateDetails, metadataSaveDetails } = - useImportSelector((state: ImportState) => state.importReducer); + const { + sampleNameColumn, + metadata, + metadataValidateDetails, + metadataSaveDetails, + } = useImportSelector((state: ImportState) => state.importReducer); const samplesUpdatedCount = metadata.filter( (metadataItem: MetadataItem) => - metadataSaveDetails[metadataItem.rowKey]?.saved === true && - metadataValidateDetails[metadataItem.rowKey].foundSampleId + metadataSaveDetails[metadataItem[sampleNameColumn]]?.saved === true && + metadataValidateDetails[metadataItem[sampleNameColumn]].foundSampleId ).length; const samplesCreatedCount = metadata.filter( (metadataItem: MetadataItem) => - metadataSaveDetails[metadataItem.rowKey]?.saved === true && - !metadataValidateDetails[metadataItem.rowKey].foundSampleId + metadataSaveDetails[metadataItem[sampleNameColumn]]?.saved === true && + !metadataValidateDetails[metadataItem[sampleNameColumn]].foundSampleId ).length; let stats = diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx index 33f5084d906..bd3d4cf6ecd 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx @@ -83,8 +83,8 @@ export function SampleMetadataImportReview(): JSX.Element { }, getCheckboxProps: (record: MetadataItem) => ({ disabled: !( - metadataValidateDetails[record.rowKey].isSampleNameValid || - metadataSaveDetails[record.rowKey]?.saved === true + metadataValidateDetails[record[sampleNameColumn]].isSampleNameValid || + metadataSaveDetails[record[sampleNameColumn]]?.saved === true ), }), }; @@ -99,7 +99,8 @@ export function SampleMetadataImportReview(): JSX.Element { React.useEffect(() => { setValid( !metadata.some( - (row) => !metadataValidateDetails[row.rowKey].isSampleNameValid + (row) => + !metadataValidateDetails[row[sampleNameColumn]].isSampleNameValid ) ); @@ -111,7 +112,8 @@ export function SampleMetadataImportReview(): JSX.Element { onCell: (item) => { return { style: { - background: metadataValidateDetails[item.rowKey].isSampleNameValid + background: metadataValidateDetails[item[sampleNameColumn]] + .isSampleNameValid ? undefined : `var(--red-1)`, }, @@ -124,10 +126,10 @@ export function SampleMetadataImportReview(): JSX.Element { fixed: "left", width: 10, render: (text, item) => { - if (metadataSaveDetails[item.rowKey]?.saved === false) + if (metadataSaveDetails[item[sampleNameColumn]]?.saved === false) return ( @@ -144,7 +146,7 @@ export function SampleMetadataImportReview(): JSX.Element { fixed: "left", width: 70, render: (text, item) => { - if (!metadataValidateDetails[item.rowKey].foundSampleId) + if (!metadataValidateDetails[item[sampleNameColumn]].foundSampleId) return ( {i18n("SampleMetadataImportReview.table.filter.new")} @@ -164,8 +166,10 @@ export function SampleMetadataImportReview(): JSX.Element { ], onFilter: (value, record) => value === "new" - ? metadataValidateDetails[record.rowKey].foundSampleId !== undefined - : metadataValidateDetails[record.rowKey].foundSampleId === undefined, + ? metadataValidateDetails[record[sampleNameColumn]].foundSampleId !== + undefined + : metadataValidateDetails[record[sampleNameColumn]].foundSampleId === + undefined, }; const otherColumns: ColumnsType = headers @@ -187,8 +191,8 @@ export function SampleMetadataImportReview(): JSX.Element { metadata .filter( (row) => - metadataValidateDetails[row.rowKey].isSampleNameValid || - metadataSaveDetails[row.rowKey]?.saved === true + metadataValidateDetails[row[sampleNameColumn]].isSampleNameValid || + metadataSaveDetails[row[sampleNameColumn]]?.saved === true ) .map((row): string => row.rowKey) ); @@ -247,7 +251,9 @@ export function SampleMetadataImportReview(): JSX.Element { className="t-metadata-uploader-review-table" rowKey={(row) => row.rowKey} rowClassName={(record) => - metadataSaveDetails[record.rowKey]?.saved === false ? "row-error" : "" + metadataSaveDetails[record[sampleNameColumn]]?.saved === false + ? "row-error" + : "" } rowSelection={rowSelection} columns={columns} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts index 415083a28be..cfda95864ed 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/services/importReducer.ts @@ -68,24 +68,23 @@ export const saveMetadata = createAsyncThunk< const metadataSaveDetails: Record = {}; const selectedSampleList = metadata.filter((metadataItem) => { - const index: string = metadataItem.rowKey; + const name: string = metadataItem[sampleNameColumn]; return ( - selectedMetadataKeys.includes(index) && - metadataSaveDetails[index]?.saved !== true + selectedMetadataKeys.includes(metadataItem.rowKey) && + metadataSaveDetails[name]?.saved !== true ); }); const chunkSize = 100; const updateSampleList = selectedSampleList .filter((metadataItem) => { - const index = metadataItem.rowKey; - const sampleId = metadataValidateDetails[index].foundSampleId; + const name = metadataItem[sampleNameColumn]; + const sampleId = metadataValidateDetails[name].foundSampleId; return sampleId; }) .map((metadataItem) => { - const index = metadataItem.rowKey; const name = metadataItem[sampleNameColumn]; - const sampleId = metadataValidateDetails[index].foundSampleId; + const sampleId = metadataValidateDetails[name].foundSampleId; const metadataFields = Object.entries(metadataItem) .filter(([key]) => headers.includes(key) && key !== sampleNameColumn) .map(([key, value]) => ({ field: key, value })); @@ -99,22 +98,30 @@ export const saveMetadata = createAsyncThunk< projectId, body: chunk, }) - .then((response) => { - // metadataSaveDetails[index] = { saved: true }; + .then(() => { + chunk + .map((item) => item.name) + .forEach((name) => (metadataSaveDetails[name] = { saved: true })); }) .catch((error) => { - // metadataSaveDetails[index] = { - // saved: false, - // error: error.response.data.error, - // }; + const { errors } = error.response.data; + Object.keys(errors).map((key) => { + metadataSaveDetails[key] = { + saved: false, + error: errors[key], + }; + }); }); + dispatch( + setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) + ); } } const createSampleList = selectedSampleList .filter((metadataItem) => { - const index = metadataItem.rowKey; - const sampleId = metadataValidateDetails[index].foundSampleId; + const name = metadataItem[sampleNameColumn]; + const sampleId = metadataValidateDetails[name].foundSampleId; return !sampleId; }) .map((metadataItem) => { @@ -132,20 +139,26 @@ export const saveMetadata = createAsyncThunk< projectId, body: chunk, }) - .then((response) => { - // metadataSaveDetails[index] = { saved: true }; + .then(() => { + chunk + .map((item) => item.name) + .forEach((name) => (metadataSaveDetails[name] = { saved: true })); }) .catch((error) => { - // metadataSaveDetails[index] = { - // saved: false, - // error: error.response.data.error, - // }; + const { errors } = error.response.data; + Object.keys(errors).map((key) => { + metadataSaveDetails[key] = { + saved: false, + error: errors[key], + }; + }); }); + dispatch( + setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) + ); } } - dispatch(setMetadataSaveDetails(Object.assign({}, metadataSaveDetails))); - return { metadataSaveDetails }; } ); @@ -177,14 +190,13 @@ export const setSampleNameColumn = createAsyncThunk< }, }); for (const metadataItem of metadata) { - const index: string = metadataItem.rowKey; - const sampleName: string = metadataItem[column]; + const name: string = metadataItem[column]; const foundSample: ValidateSampleNameModel | undefined = response.samples.find( - (sample: ValidateSampleNameModel) => sampleName === sample.name + (sample: ValidateSampleNameModel) => name === sample.name ); - metadataValidateDetails[index] = { - isSampleNameValid: validateSampleName(sampleName), + metadataValidateDetails[name] = { + isSampleNameValid: validateSampleName(name), foundSampleId: foundSample?.ids?.at(0), }; } From e946a5a88a78508c6502d70d4a41ba92899bbf8d Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 05:20:47 -0600 Subject: [PATCH 371/655] chore: Debounced search and param updates. Default column to modified date --- .../js/pages/search/SearchLayout.tsx | 36 ++++++++++++++++--- .../js/pages/search/SearchProjectsTable.tsx | 8 ++++- .../js/pages/search/SearchSamplesTable.tsx | 7 +++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index 8106d038460..7087274ad73 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -18,6 +18,7 @@ import axios from "axios"; import { setBaseUrl } from "../../utilities/url-utilities"; import { Sample } from "../../types/irida"; import { CurrentUser } from "../../apis/users/user"; +import { debounce } from "lodash"; type SearchType = "projects" | "samples"; type SearchItem = { @@ -58,6 +59,9 @@ const initial_table_params = JSON.stringify({ export default function SearchLayout() { const user = useLoaderData() as CurrentUser; const [searchParams, setSearchParams] = useSearchParams(); + const debouncedSetSearchParams = debounce(async (value) => { + setSearchParams(value); + }, 500); const [type, setType] = useState("projects"); const [global, setGlobal] = useState(user.admin); @@ -68,6 +72,9 @@ export default function SearchLayout() { const [projectsTableParams, setProjectsTableParams] = useState( JSON.parse(initial_table_params) ); + const debouncedSetProjectTableParams = debounce(async (params) => { + setProjectsTableParams(params); + }, 300); const [samples, setSamples] = useState<{ content: SearchSample[]; @@ -76,13 +83,16 @@ export default function SearchLayout() { const [samplesTableParams, setSamplesTableParams] = useState( JSON.parse(initial_table_params) ); + const debouncedSetSamplesTableParams = debounce(async (params) => { + setSamplesTableParams(params); + }, 300); const handleProjectsTableChange = ( pagination: TablePaginationConfig, filters: Record, sorter: SorterResult ) => { - setProjectsTableParams({ + debouncedSetProjectTableParams({ pagination, filters, ...sorter, @@ -94,7 +104,7 @@ export default function SearchLayout() { filters: Record, sorter: SorterResult ) => { - setSamplesTableParams({ + debouncedSetSamplesTableParams({ pagination, filters, ...sorter, @@ -106,7 +116,12 @@ export default function SearchLayout() { axios.post(setBaseUrl(`/ajax/search/samples`), { global, pagination: samplesTableParams.pagination, - order: [{ property: "sampleName", direction: `asc` }], + order: [ + { + property: samplesTableParams.columnKey || `sampleName`, + direction: samplesTableParams.order === "ascend" ? `asc` : `desc`, + }, + ], search: [ { property: `name`, @@ -120,7 +135,12 @@ export default function SearchLayout() { axios.post(setBaseUrl(`/ajax/search/projects`), { global, pagination: projectsTableParams.pagination, - order: [{ property: "name", direction: `asc` }], + order: [ + { + property: projectsTableParams.columnKey || `name`, + direction: projectsTableParams.order === "ascend" ? `asc` : `desc`, + }, + ], search: [ { property: `name`, @@ -141,7 +161,11 @@ export default function SearchLayout() { ); }, [ global, + projectsTableParams.columnKey, + projectsTableParams.order, projectsTableParams.pagination, + samplesTableParams.columnKey, + samplesTableParams.order, samplesTableParams.pagination, searchParams, ]); @@ -232,7 +256,9 @@ export default function SearchLayout() { addonBefore={user.admin && searchPrefix} size={"large"} defaultValue={searchParams.get("query") || ""} - onChange={(e) => setSearchParams({ query: e.target.value })} + onChange={(e) => + debouncedSetSearchParams({ query: e.target.value }) + } allowClear /> diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index 51379a4f2c4..1f68581f170 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -6,6 +6,7 @@ import { Table, TablePaginationConfig } from "antd"; import { FilterValue, SorterResult } from "antd/es/table/interface"; import { SampleTableType, SearchProject, TableParams } from "./SearchLayout"; import ProjectTag from "./ProjectTag"; +import { ColumnsType } from "antd/lib/table"; type SearchProjectsTableParams = { projects: @@ -24,18 +25,20 @@ export default function SearchProjectsTable({ projects, handleTableChange, }: SearchProjectsTableParams) { - const columns = useMemo( + const columns = useMemo( () => [ { key: `name`, dataIndex: "name", title: "NAME", render: (_, project) => , + sorter: true, }, { key: `organism`, dataIndex: `organism`, title: "ORGANISM", + sorter: true, }, { key: `samples`, @@ -47,12 +50,15 @@ export default function SearchProjectsTable({ dataIndex: `createdDate`, title: `CREATED DATE`, render: (text: string) => formatInternationalizedDateTime(text), + sorter: true, }, { key: `modifiedDate`, dataIndex: `modifiedDate`, title: `MODIFIED DATE`, render: (text: string) => formatInternationalizedDateTime(text), + sorter: true, + defaultSortOrder: "descend", }, ], [] diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx index 132bcd87bc7..24dc5fb0604 100644 --- a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -28,14 +28,16 @@ export default function SearchSamplesTable({ const columns = useMemo( () => [ { - key: `name`, + key: `sampleName`, dataIndex: "name", title: "NAME", + sorter: true, }, { key: `organism`, dataIndex: `organism`, title: "ORGANISM", + sorter: true, }, { key: `projects`, @@ -50,12 +52,15 @@ export default function SearchSamplesTable({ dataIndex: `createdDate`, title: `CREATED DATE`, render: (text: string) => formatInternationalizedDateTime(text), + sorter: true, }, { key: `modifiedDate`, dataIndex: `modifiedDate`, title: `MODIFIED DATE`, render: (text: string) => formatInternationalizedDateTime(text), + sorter: true, + defaultSortOrder: "descend", }, ], [] From 09ce0867a479b1c448d31bbe4a3a9ef81c25a19a Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 05:30:25 -0600 Subject: [PATCH 372/655] chore: Added fixed widths to modified and created date columns --- .../webapp/resources/js/pages/search/SearchProjectsTable.tsx | 3 +++ .../webapp/resources/js/pages/search/SearchSamplesTable.tsx | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index 1f68581f170..3c017290887 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -44,6 +44,7 @@ export default function SearchProjectsTable({ key: `samples`, dataIndex: `samples`, title: `SAMPLES`, + width: 150, }, { key: `createdDate`, @@ -51,6 +52,7 @@ export default function SearchProjectsTable({ title: `CREATED DATE`, render: (text: string) => formatInternationalizedDateTime(text), sorter: true, + width: 200, }, { key: `modifiedDate`, @@ -59,6 +61,7 @@ export default function SearchProjectsTable({ render: (text: string) => formatInternationalizedDateTime(text), sorter: true, defaultSortOrder: "descend", + width: 200, }, ], [] diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx index 24dc5fb0604..1cb588350f0 100644 --- a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -53,6 +53,7 @@ export default function SearchSamplesTable({ title: `CREATED DATE`, render: (text: string) => formatInternationalizedDateTime(text), sorter: true, + width: 200, }, { key: `modifiedDate`, @@ -61,6 +62,7 @@ export default function SearchSamplesTable({ render: (text: string) => formatInternationalizedDateTime(text), sorter: true, defaultSortOrder: "descend", + width: 200, }, ], [] From 9e842f48a3c83bc0ee5698c3f7100c93ac868e3e Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 05:34:06 -0600 Subject: [PATCH 373/655] doc: Updated JavaDoc for SearchController --- .../ria/web/search/SearchController.java | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java index d54d5c776df..843c940043b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/SearchController.java @@ -4,9 +4,6 @@ import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; import ca.corefacility.bioinformatics.irida.model.project.Project; import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesResponse; -import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProject; -import ca.corefacility.bioinformatics.irida.ria.web.models.datatables.DTProjectSamples; import ca.corefacility.bioinformatics.irida.ria.web.models.tables.AntTableResponse; import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchItem; import ca.corefacility.bioinformatics.irida.ria.web.search.dto.SearchProject; @@ -55,10 +52,8 @@ public String search() { /** * Search all projects a user is a member of based on a query string * - * @param query the query string - * @param global Whether to perform an admin global search - * @param params parameters for a datatables response - * @return a {@link DataTablesResponse} to display the search results + * @param request Details about the search request + * @return Paged list of projects and the total count found for the search */ @PostMapping("/ajax/search/projects") public ResponseEntity> handleSearch(@RequestBody SearchRequest request) { @@ -78,11 +73,8 @@ public ResponseEntity> handleSearch(@RequestBody Se /** * Search all {@link Sample}s in projects for a user based on a query string * - * @param query the query string - * @param global Whether to perform an admin - * global search - * @param params parameters for a datatables response - * @return a {@link DataTablesResponse} to display search results + * @param request Details about the search request + * @return Paged list of samples and the total count found in the search */ @RequestMapping("/ajax/search/samples") @ResponseBody @@ -112,26 +104,4 @@ public ResponseEntity> searchSamples(@RequestBody S AntTableResponse response = new AntTableResponse<>(samples, samplePage.getTotalElements()); return ResponseEntity.ok(response); } - - /** - * Extract the details of the a {@link Project} into a {@link DTProject} - * which is consumable by the UI - * - * @param project {@link Project} - * @return {@link DTProject} - */ - private DTProject createDataTablesProject(Project project) { - return new DTProject(project, sampleService.getNumberOfSamplesForProject(project)); - } - - /** - * Extract the details of a {@link ProjectSampleJoin} into a - * {@link DTProjectSamples} - * - * @param join the {@link ProjectSampleJoin} - * @return the created {@link DTProjectSamples} - */ - private DTProjectSamples createDataTablesSample(ProjectSampleJoin join) { - return new DTProjectSamples(join, Lists.newArrayList(), null); - } } From ae960cbc2317d61326bdb153022ab69b7a11a57c Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 05:35:24 -0600 Subject: [PATCH 374/655] chore: Updated JSDoc --- src/main/webapp/resources/js/pages/search/index.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/webapp/resources/js/pages/search/index.tsx b/src/main/webapp/resources/js/pages/search/index.tsx index 02d888344e9..0a08f6b86f8 100644 --- a/src/main/webapp/resources/js/pages/search/index.tsx +++ b/src/main/webapp/resources/js/pages/search/index.tsx @@ -11,6 +11,11 @@ import { setBaseUrl } from "../../utilities/url-utilities"; import userLoader from "./loaders/user-loader"; import SearchLayout from "./SearchLayout"; +/** + * @fileoverview + * Sets up the base layout and router for the global search page + */ + const router = createBrowserRouter( createRoutesFromElements( Date: Thu, 10 Nov 2022 07:49:36 -0600 Subject: [PATCH 375/655] chore: Made component for displaying the search counts in side navigation --- .../resources/js/pages/search/SearchCount.tsx | 13 +++++++++++++ .../resources/js/pages/search/SearchLayout.tsx | 15 +++++---------- .../js/pages/search/SearchProjectsTable.tsx | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 src/main/webapp/resources/js/pages/search/SearchCount.tsx diff --git a/src/main/webapp/resources/js/pages/search/SearchCount.tsx b/src/main/webapp/resources/js/pages/search/SearchCount.tsx new file mode 100644 index 00000000000..af99bfabf13 --- /dev/null +++ b/src/main/webapp/resources/js/pages/search/SearchCount.tsx @@ -0,0 +1,13 @@ +import React from "react"; +import { Badge } from "antd"; + +export default function SearchCount({ count }: { count: number }) { + return ( + + ); +} diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index 7087274ad73..f91ab008aea 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -19,6 +19,7 @@ import { setBaseUrl } from "../../utilities/url-utilities"; import { Sample } from "../../types/irida"; import { CurrentUser } from "../../apis/users/user"; import { debounce } from "lodash"; +import SearchCount from "./SearchCount"; type SearchType = "projects" | "samples"; type SearchItem = { @@ -36,12 +37,14 @@ export type SearchProject = SearchItem & { export type SearchSample = SearchItem & { projects: SearchProject[]; }; + export type SampleTableType = Pick< Sample, "name" | "organism" | "projects" | "createdDate" | "modifiedDate" > & { key: "string"; }; + export interface TableParams { pagination?: TablePaginationConfig; sortField?: string; @@ -187,11 +190,7 @@ export default function SearchLayout() { }} > PROJECTS - + ), key: "projects", @@ -207,11 +206,7 @@ export default function SearchLayout() { }} > SAMPLES - + ), key: "samples", diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index 3c017290887..f81ab1d3d99 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -57,7 +57,7 @@ export default function SearchProjectsTable({ { key: `modifiedDate`, dataIndex: `modifiedDate`, - title: `MODIFIED DATE`, + title: `LAST UPDATED`, render: (text: string) => formatInternationalizedDateTime(text), sorter: true, defaultSortOrder: "descend", From 228904192f86b1816f04325d594dee2c2d797346 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 08:11:16 -0600 Subject: [PATCH 376/655] chore: Cleaned up types and JSDoc --- .../js/pages/search/SearchLayout.tsx | 28 +++++++++--------- .../js/pages/search/SearchProjectsTable.tsx | 21 +++++++------- .../js/pages/search/SearchSamplesTable.tsx | 29 ++++++++++--------- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index f91ab008aea..845c233edbf 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -3,7 +3,6 @@ import { useLoaderData, useSearchParams } from "react-router-dom"; import SearchProjectsTable from "./SearchProjectsTable"; import SearchSamplesTable from "./SearchSamplesTable"; import { - Badge, Input, Layout, Menu, @@ -11,9 +10,10 @@ import { Select, Space, TablePaginationConfig, + TableProps, Typography, } from "antd"; -import { FilterValue, SorterResult } from "antd/es/table/interface"; +import { FilterValue } from "antd/es/table/interface"; import axios from "axios"; import { setBaseUrl } from "../../utilities/url-utilities"; import { Sample } from "../../types/irida"; @@ -25,7 +25,7 @@ type SearchType = "projects" | "samples"; type SearchItem = { id: number; name: string; - createdDate: number; + createdDate: string; modifiedDate: number; organism: string; }; @@ -50,6 +50,8 @@ export interface TableParams { sortField?: string; sortOrder?: string; filters?: Record; + columnKey?: string; + order: "ascend" | "descend"; } const initial_table_params = JSON.stringify({ @@ -90,11 +92,11 @@ export default function SearchLayout() { setSamplesTableParams(params); }, 300); - const handleProjectsTableChange = ( - pagination: TablePaginationConfig, - filters: Record, - sorter: SorterResult - ) => { + const handleProjectsTableChange: TableProps["onChange"] = ( + pagination, + filters, + sorter + ): void => { debouncedSetProjectTableParams({ pagination, filters, @@ -102,11 +104,11 @@ export default function SearchLayout() { }); }; - const handleSamplesTableChange = ( - pagination: TablePaginationConfig, - filters: Record, - sorter: SorterResult - ) => { + const handleSamplesTableChange: TableProps["onChange"] = ( + pagination, + filters, + sorter + ): void => { debouncedSetSamplesTableParams({ pagination, filters, diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index f81ab1d3d99..0e6268bb109 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -1,10 +1,8 @@ import React, { useMemo } from "react"; -import type { ColumnType } from "antd/es/list"; import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; -import { Table, TablePaginationConfig } from "antd"; -import { FilterValue, SorterResult } from "antd/es/table/interface"; -import { SampleTableType, SearchProject, TableParams } from "./SearchLayout"; +import { Table, TableProps } from "antd"; +import { SearchProject } from "./SearchLayout"; import ProjectTag from "./ProjectTag"; import { ColumnsType } from "antd/lib/table"; @@ -15,17 +13,20 @@ type SearchProjectsTableParams = { total: number; } | undefined; - handleTableChange: ( - pagination: TablePaginationConfig, - filters: Record, - sorter: SorterResult - ) => TableParams; + handleTableChange: TableProps["onChange"]; }; + +/** + * React component to render a table of projects fround in the glbal search. + * @param projects + * @param handleTableChange + * @constructor + */ export default function SearchProjectsTable({ projects, handleTableChange, }: SearchProjectsTableParams) { - const columns = useMemo( + const columns = useMemo>( () => [ { key: `name`, diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx index 1cb588350f0..51686ca1ffb 100644 --- a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -1,12 +1,10 @@ import React, { useMemo } from "react"; -import type { ColumnType } from "antd/es/list"; import { formatInternationalizedDateTime } from "../../utilities/date-utilities"; import { getPaginationOptions } from "../../utilities/antdesign-table-utilities"; -import { Table, TablePaginationConfig, Tag } from "antd"; -import { FilterValue, SorterResult } from "antd/es/table/interface"; -import { TableParams } from "./index"; -import { SampleTableType, SearchSample } from "./SearchLayout"; +import { Table, TableProps } from "antd"; +import { SearchProject, SearchSample } from "./SearchLayout"; import ProjectTag from "./ProjectTag"; +import { ColumnsType } from "antd/lib/table"; type SearchSamplesTableParams = { samples: @@ -15,17 +13,20 @@ type SearchSamplesTableParams = { total: number; } | undefined; - handleTableChange: ( - pagination: TablePaginationConfig, - filters: Record, - sorter: SorterResult - ) => TableParams; + handleTableChange: TableProps["onChange"]; }; + +/** + * React component to render a table to display samples found in global search + * @param samples + * @param handleTableChange + * @constructor + */ export default function SearchSamplesTable({ samples, handleTableChange, }: SearchSamplesTableParams) { - const columns = useMemo( + const columns = useMemo>( () => [ { key: `sampleName`, @@ -43,8 +44,10 @@ export default function SearchSamplesTable({ key: `projects`, dataIndex: `projects`, title: `PROJECTS`, - render: (projects) => { - return projects.map((project) => ); + render: (projects: SearchProject[]) => { + return projects.map((project) => ( + + )); }, }, { From 8d630ab859453caa05aa52870795b58575d94772 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 08:12:46 -0600 Subject: [PATCH 377/655] chore: Updated JavaDoc --- .../bioinformatics/irida/ria/web/search/dto/SearchRequest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java index 15674a286c1..909c74be245 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchRequest.java @@ -2,6 +2,9 @@ import ca.corefacility.bioinformatics.irida.ria.web.models.tables.AntTableRequest; +/** + * Search request object for the global search table. + */ public class SearchRequest extends AntTableRequest { private boolean global; From dda7de8ba8d22bb0235fe1d1c31a97862890bec3 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 08:13:26 -0600 Subject: [PATCH 378/655] chore: Updated JavaDoc --- .../bioinformatics/irida/ria/web/search/dto/SearchSample.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java index c393b729cae..a89ea655049 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchSample.java @@ -4,6 +4,9 @@ import java.util.List; +/** + * Data transfer object for {@link Sample}s found in the global search + */ public class SearchSample extends SearchItem { final String organism; final List projects; From 040bb12133fad2d86f47577d56d503dc3bea3ef1 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 08:13:57 -0600 Subject: [PATCH 379/655] chore: Updated JavaDoc --- .../bioinformatics/irida/ria/web/search/dto/SearchProject.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java index 1e76fb5302a..260d1ce71aa 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/search/dto/SearchProject.java @@ -2,6 +2,9 @@ import ca.corefacility.bioinformatics.irida.model.project.Project; +/** + * Data transfer object for {@link Project}s found in the global search + */ public class SearchProject extends SearchItem { final String organism; final Long samples; From effa040121501f9e0cfc471c6e47e59e5cd30167 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 08:14:58 -0600 Subject: [PATCH 380/655] chore: Updated JSDoc --- src/main/webapp/resources/js/pages/search/SearchCount.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/webapp/resources/js/pages/search/SearchCount.tsx b/src/main/webapp/resources/js/pages/search/SearchCount.tsx index af99bfabf13..6d4beae8a79 100644 --- a/src/main/webapp/resources/js/pages/search/SearchCount.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchCount.tsx @@ -1,6 +1,11 @@ import React from "react"; import { Badge } from "antd"; +/** + * React component to display the total number of items found in the global search + * @param count + * @constructor + */ export default function SearchCount({ count }: { count: number }) { return ( Date: Thu, 10 Nov 2022 08:16:13 -0600 Subject: [PATCH 381/655] chore: Updated JSDoc --- src/main/webapp/resources/js/pages/search/SearchLayout.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index 845c233edbf..2e1209b5ae5 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -61,6 +61,10 @@ const initial_table_params = JSON.stringify({ }, }); +/** + * React component to layout and handle events for the global search + * @constructor + */ export default function SearchLayout() { const user = useLoaderData() as CurrentUser; const [searchParams, setSearchParams] = useSearchParams(); From fa24cf0f6b05d89c96e28d0f9fb951887a621a52 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 08:22:48 -0600 Subject: [PATCH 382/655] chore: Removed DataTables and font-awesome from frontend --- src/main/webapp/package.json | 7 - src/main/webapp/pnpm-lock.yaml | 63 ----- src/main/webapp/resources/js/app.js | 2 - .../js/utilities/datatables-utilities.js | 219 ------------------ .../js/utilities/fontawesome-utilities.js | 24 -- .../js/vendor/datatables/datatables.js | 4 - 6 files changed, 319 deletions(-) delete mode 100644 src/main/webapp/resources/js/utilities/datatables-utilities.js delete mode 100644 src/main/webapp/resources/js/utilities/fontawesome-utilities.js delete mode 100644 src/main/webapp/resources/js/vendor/datatables/datatables.js diff --git a/src/main/webapp/package.json b/src/main/webapp/package.json index a410d3ece25..eb7d8911168 100644 --- a/src/main/webapp/package.json +++ b/src/main/webapp/package.json @@ -26,7 +26,6 @@ "@antv/path-util": "^2.0.15", "@antv/scale": "^0.3.17", "@antv/util": "^2.0.17", - "@fortawesome/fontawesome-free": "^6.1.1", "@loaders.gl/core": ">=3.2.0 <4.0.0", "@loaders.gl/gltf": "^3.0.0", "@loaders.gl/images": ">=3.0.0 <4.0.0", @@ -43,12 +42,6 @@ "bootstrap": "3.4.1", "bootstrap-daterangepicker": "^3.0.3", "clipboard": "^1.7.1", - "datatables.net": "^1.11.5", - "datatables.net-bs": "^1.11.5", - "datatables.net-buttons": "^2.2.2", - "datatables.net-buttons-bs": "^2.2.2", - "datatables.net-fixedcolumns": "^4.0.2", - "datatables.net-fixedcolumns-bs": "^4.0.2", "dayjs": "^1.11.1", "deck.gl": "^8.7.12", "flexlayout-react": "^0.7.4", diff --git a/src/main/webapp/pnpm-lock.yaml b/src/main/webapp/pnpm-lock.yaml index d60f1acf1d9..fc0612befb8 100644 --- a/src/main/webapp/pnpm-lock.yaml +++ b/src/main/webapp/pnpm-lock.yaml @@ -25,7 +25,6 @@ specifiers: '@babel/preset-react': ^7.18.6 '@babel/preset-typescript': ^7.18.6 '@babel/runtime': ^7.20.1 - '@fortawesome/fontawesome-free': ^6.1.1 '@loaders.gl/core': '>=3.2.0 <4.0.0' '@loaders.gl/gltf': ^3.0.0 '@loaders.gl/images': '>=3.0.0 <4.0.0' @@ -59,12 +58,6 @@ specifiers: clipboard: ^1.7.1 css-loader: ^6.7.1 css-minimizer-webpack-plugin: ^3.4.1 - datatables.net: ^1.11.5 - datatables.net-bs: ^1.11.5 - datatables.net-buttons: ^2.2.2 - datatables.net-buttons-bs: ^2.2.2 - datatables.net-fixedcolumns: ^4.0.2 - datatables.net-fixedcolumns-bs: ^4.0.2 dayjs: ^1.11.1 deck.gl: ^8.7.12 eslint: ^8.13.0 @@ -135,7 +128,6 @@ dependencies: '@antv/path-util': 2.0.15 '@antv/scale': 0.3.17 '@antv/util': 2.0.17 - '@fortawesome/fontawesome-free': 6.1.1 '@loaders.gl/core': 3.2.4 '@loaders.gl/gltf': 3.2.4 '@loaders.gl/images': 3.2.4 @@ -152,12 +144,6 @@ dependencies: bootstrap: 3.4.1 bootstrap-daterangepicker: 3.1.0 clipboard: 1.7.1 - datatables.net: 1.11.5 - datatables.net-bs: 1.11.5 - datatables.net-buttons: 2.2.2 - datatables.net-buttons-bs: 2.2.2 - datatables.net-fixedcolumns: 4.0.2 - datatables.net-fixedcolumns-bs: 4.0.2 dayjs: 1.11.1 deck.gl: 8.7.12_qdym3cmqy3izoldsblxf2ftyey flexlayout-react: 0.7.4_sfoxds7t5ydpegc3knd667wn6m @@ -2090,12 +2076,6 @@ packages: - supports-color dev: true - /@fortawesome/fontawesome-free/6.1.1: - resolution: {integrity: sha512-J/3yg2AIXc9wznaVqpHVX3Wa5jwKovVF0AMYSnbmcXTiL3PpRPfF58pzWucCwEiCJBp+hCNRLWClTomD8SseKg==} - engines: {node: '>=6'} - requiresBuild: true - dev: false - /@humanwhocodes/config-array/0.9.3: resolution: {integrity: sha512-3xSMlXHh03hCcCmFc0rbKp3Ivt2PFEJnQUJDDMTJQ2wkECZWdq4GePs2ctc5H8zV+cHPaq8k2vU8mrQjA6iHdQ==} engines: {node: '>=10.10.0'} @@ -4511,49 +4491,6 @@ packages: whatwg-url: 8.7.0 dev: true - /datatables.net-bs/1.11.5: - resolution: {integrity: sha512-thj90yxWnJpAkKdiPtdFMHlDTHCg1VRrzAE7juABYaw3hNQdvf1Veh7a/1hvuqOfn2g7ORzQUFXP8oxJ4SJWfg==} - dependencies: - datatables.net: 1.11.5 - jquery: 3.5.1 - dev: false - - /datatables.net-buttons-bs/2.2.2: - resolution: {integrity: sha512-F4ffoweXJrtznq+KfYYW7aa1xHlzQUtSEJXne7rUvp76neMmuFtNhD+2ffaQHizNJmiyGAEKgq54pD8qkGo63A==} - dependencies: - datatables.net-bs: 1.11.5 - datatables.net-buttons: 2.2.2 - jquery: 3.5.1 - dev: false - - /datatables.net-buttons/2.2.2: - resolution: {integrity: sha512-YrTgBDejFSzm2CJOx0aOACXDIELBsdMFe6IR3rJqC0elJoMd6uXVQ4pRBQEge4cfWpYa5T1Majm7r6FQi0j6Jw==} - dependencies: - datatables.net: 1.11.5 - jquery: 3.5.1 - dev: false - - /datatables.net-fixedcolumns-bs/4.0.2: - resolution: {integrity: sha512-DdOZy1WN9V3hF7C9jzPspjC9VEnKdeP225jDs5O8/ZYHrEM2MHOuUVKIpNjikICi4ABQFJPd6pMKDXqvQLvKHQ==} - dependencies: - datatables.net-bs: 1.11.5 - datatables.net-fixedcolumns: 4.0.2 - jquery: 3.5.1 - dev: false - - /datatables.net-fixedcolumns/4.0.2: - resolution: {integrity: sha512-BOLffsEyCy/uKTubVf1GAV/S/PEuqTE8up+E55zROcsciwFy6C2VMx+yGaaJAhAZTgFf0zneb55tHIaoFXNnfQ==} - dependencies: - datatables.net: 1.11.5 - jquery: 3.5.1 - dev: false - - /datatables.net/1.11.5: - resolution: {integrity: sha512-nlFst2xfwSWaQgaOg5sXVG3cxYC0tH8E8d65289w9ROgF2TmLULOOpcdMpyxxUim/qEwVSEem42RjkTWEpr3eA==} - dependencies: - jquery: 3.5.1 - dev: false - /date-fns/2.28.0: resolution: {integrity: sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw==} engines: {node: '>=0.11'} diff --git a/src/main/webapp/resources/js/app.js b/src/main/webapp/resources/js/app.js index edb9bacd5bf..df0b2cd6b90 100644 --- a/src/main/webapp/resources/js/app.js +++ b/src/main/webapp/resources/js/app.js @@ -1,7 +1,5 @@ // Import css import "../css/app.css"; -// Font Awesome -import "@fortawesome/fontawesome-free/js/all"; /* This will load notifications into the global namespace. Remove this once all files have been converted over to wekbpack builds. diff --git a/src/main/webapp/resources/js/utilities/datatables-utilities.js b/src/main/webapp/resources/js/utilities/datatables-utilities.js deleted file mode 100644 index 24785795a3b..00000000000 --- a/src/main/webapp/resources/js/utilities/datatables-utilities.js +++ /dev/null @@ -1,219 +0,0 @@ -import $ from "jquery"; -import { addTooltip } from "./bootstrap-utilities"; -import { createIcon, ICONS } from "./fontawesome-utilities"; -import snakeCase from "lodash/snakeCase"; - -/* - Default layout around DataTables in IRIDA. - @link https://datatables.net/reference/option/dom - dom expands into: -
    -
    - [BUTTONS] - All DataTables specific buttons get loaded here. -
    -
    - [FILTER] <-- This is the DataTables search box (and button to open advanced filters if needed) -
    -
    -
    -
    - [FILTER TAGS] (for quickly removing filters) go here. -
    -
    -[PROCESSING] - Displayed when DataTables is fetching and processing new data. -[TABLE] - Actual table -
    -
    - [LENGTH] - length changing control input -
    -
    - [PAGING] - pagination control -
    -
    - [INFO] - table information summary -
    -
    -*/ -const dom = ` -<".row" - <"col-md-6 col-sm-12 buttons"B><"#dt-filters.col-md-6 col-sm-12"f>> -<".row"<"col-md-3 selected-counts"> - <"col-md-9 filter-tags"<"filter-tags__space">> -> -<"dt-table-wrapper"rt> -<"row" - <"col-md-3 col-sm-12"l> - <" col-md-6 col-sm-12"p><"col-md-3 col-sm-12 text-right"i>>`; - -/** - * Default DataTables configuration object. Anything can be overwritten, - * but this will provide a baseline for all DataTables. - */ -export const tableConfig = { - dom, - processing: true, - serverSide: true, - deferRender: true, - createdRow(row) { - $(row).tooltip({ selector: "[data-toggle='tooltip']" }); - } -}; - -/** - * Create a button to download a file. - * @param {string} url of the item to download - * @param {title} title to label the download by. - * @return {object} DOM anchor element for download. - */ -export function createDownloadLink({ url, title }) { - const anchor = document.createElement("a"); - anchor.classList.add("btn", "btn-default", "download-btn"); - anchor.download = title; - anchor.setAttribute("href", url); - addTooltip({ dom: anchor, title: "Download" }); - - const icon = createIcon({ icon: ICONS.download, fixed: true }); - anchor.append(icon); - return anchor; -} - -/** - * Create a button to delete a row. - * @param {object} data attributes need to delete an item. - * @return {object} DOM element for button. - */ -export function createDeleteBtn(data = { title: "Delete" }) { - const btn = document.createElement("button"); - btn.classList.add("btn", "btn-default", "remove-btn"); - // Add any required data attributes. - Object.assign(btn.dataset, data); - - const icon = createIcon({ icon: ICONS.trash, fixed: true }); - const tooltiped = addTooltip({ dom: icon, title: "Delete" }); - btn.append(tooltiped); - return btn; -} - -/** - * Create a button group containing action buttons for this row. - * @param {array} buttons list of DOM nodes for buttons. - * @return {*} Either the button group or an empty string. - */ -export function createButtonCell(buttons = []) { - if (buttons.length) { - const wrapper = $(`
    -
    -
    `); - const btns = wrapper.find(".btn-group"); - for (const btn of buttons) { - btns.append(btn); - } - return wrapper.html(); - } - return ""; -} - -/** - * Create an anchor tag to link to an item. - * @param {string} url to find the item at - * @param {string} label label for the item - * @param {string} width width for the button - * @param {array} list of extra classes to add to the link - * @return {*} anchor element containing link. - */ -export function createItemLink({ url, label, width, classes = [] }) { - if (typeof url !== "undefined" && typeof label !== "undefined") { - const link = document.createElement("a"); - link.classList.add("btn", "btn-link", "dt-wrap-cell", ...classes); - link.style.width = width; - link.style.textAlign = "left"; - link.href = url; - link.innerHTML = label; - - return link.outerHTML; - } - return label || ""; -} - -/** - * Get the order of the columns on the page. - * @return {object} {{COLUMN_NAME: index}} - */ -export function generateColumnOrderInfo(tableId) { - let selector = "thead th"; - if (tableId) { - selector = tableId + " " + selector; - } - const columns = {}; - $(selector).each((index, elm) => { - const data = snakeCase($(elm).data("data")).toUpperCase(); - columns[data] = index; - }); - return columns; -} - -/** - * Make the content of a column a fixed width. This will add an ellipsis to the text - * if too long. - * @param {string} text content - * @param {number} width for the column - * @return {Element} formatted DOM as text - */ -export function createRestrictedWidthContent({ text, width = 150 }) { - const dom = document.createElement("div"); - dom.classList.add("cell-restricted"); - dom.style.width = `${width}px`; - dom.innerText = text; - return addTooltip({ dom, title: text }); -} - -/** - * Wrap the contents of a cell in a div that will force wrapping at the desired width. - * @param {string} text to put in cell - * @param {string} width desired (e.g. '100px' or '50em') - * @return {string} formatted DOM as text - */ -export function wrapCellContents({ text, width = "250px" }) { - const div = document.createElement("div"); - div.classList.add("dt-wrap-cell"); - div.style.width = width; - div.innerHTML = text; - return div.outerHTML; -} - -/** - * Create a filter tag on the datatable - * @param {string} text value searched - * @param {string} type field searched - * @param {function} handler what to do when button is clicked - */ -export function createFilterTag({ text, type, handler }) { - const remove = elm => { - $(elm) - .off("click") - .remove(); - }; - const $filterTags = $(".filter-tags"); - - // Check to see if that button already exists. - const btn = $filterTags.find(`[data-type="${type}"]`); - if (btn) { - remove(btn); - } - - if (text) { - const tag = ` - `; - const $tag = $(tag); - $tag.on("click", function() { - // Remove tag - remove(this); - handler(); - }); - $filterTags.append($tag); - } -} diff --git a/src/main/webapp/resources/js/utilities/fontawesome-utilities.js b/src/main/webapp/resources/js/utilities/fontawesome-utilities.js deleted file mode 100644 index 094a19c7537..00000000000 --- a/src/main/webapp/resources/js/utilities/fontawesome-utilities.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * This file is for create dynamic font-awesome icons. - */ - -export const ICONS = { - download: "fa-download", // Downloading, - trash: "fa-trash", // Deleting - remove: "fa-times", // Removing - edit: "fa-pencil", // Editing - checkmark: "fa-check" -}; - -/** - * Create a font-awesome icon within an `i` DOM element - * @param {string} icon to display - * @param {boolean} fixed width - * @return {Element} icon element - */ -export function createIcon({ icon = "", fixed = false }) { - const i = ``; - const div = document.createElement("div"); - div.innerHTML = i; - return div.childNodes[0]; -} diff --git a/src/main/webapp/resources/js/vendor/datatables/datatables.js b/src/main/webapp/resources/js/vendor/datatables/datatables.js deleted file mode 100644 index bf2ee4f50b1..00000000000 --- a/src/main/webapp/resources/js/vendor/datatables/datatables.js +++ /dev/null @@ -1,4 +0,0 @@ -import "datatables.net"; -import "datatables.net-bs"; -import "datatables.net-bs/css/dataTables.bootstrap.css"; -import "../../../css/components/datatables.css"; \ No newline at end of file From 381bd1a77c1780dd0a2b24f43b1a53584d19aed2 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 09:37:00 -0600 Subject: [PATCH 383/655] chore: Updated browserlist --- src/main/webapp/package.json | 24 +++++++-------- src/main/webapp/pnpm-lock.yaml | 53 +++++++++++++++++----------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/main/webapp/package.json b/src/main/webapp/package.json index eb7d8911168..4c0f8e421b1 100644 --- a/src/main/webapp/package.json +++ b/src/main/webapp/package.json @@ -26,13 +26,13 @@ "@antv/path-util": "^2.0.15", "@antv/scale": "^0.3.17", "@antv/util": "^2.0.17", - "@loaders.gl/core": ">=3.2.0 <4.0.0", - "@loaders.gl/gltf": "^3.0.0", - "@loaders.gl/images": ">=3.0.0 <4.0.0", - "@luma.gl/engine": ">=8.4.0 <9.0.0", - "@luma.gl/gltools": ">=8.4.0 <9.0.0", - "@luma.gl/shadertools": "^8.4.0", - "@luma.gl/webgl": ">=8.4.0 <9.0.0", + "@loaders.gl/core": "^3.2.4", + "@loaders.gl/gltf": "^3.2.4", + "@loaders.gl/images": "^3.2.4", + "@luma.gl/engine": "^8.5.14", + "@luma.gl/gltools": "^8.5.14", + "@luma.gl/shadertools": "^8.5.14", + "@luma.gl/webgl": "^8.5.14", "@phylocanvas/phylocanvas.gl": "^1.43.0", "@reduxjs/toolkit": "1.8.5", "ag-grid-community": "^27.1.0", @@ -40,12 +40,12 @@ "antd": "4.19.5", "axios": "^0.26.1", "bootstrap": "3.4.1", - "bootstrap-daterangepicker": "^3.0.3", + "bootstrap-daterangepicker": "^3.1.0", "clipboard": "^1.7.1", "dayjs": "^1.11.1", "deck.gl": "^8.7.12", "flexlayout-react": "^0.7.4", - "gl-matrix": "^3.0.0", + "gl-matrix": "^3.4.3", "i": "^0.3.7", "immutability-helper": "^3.1.1", "jquery": "3.5.1", @@ -88,8 +88,8 @@ "@types/lodash": "^4.14.182", "@types/lodash.uniqby": "^4.7.7", "@types/node": "^17.0.39", - "@types/react": "^17.0.39", - "@types/react-dom": "^17.0.11", + "@types/react": "^17.0.45", + "@types/react-dom": "^17.0.17", "@types/react-redux": "^7.1.24", "@types/react-virtualized-auto-sizer": "^1.0.1", "@types/react-window": "^1.8.5", @@ -117,7 +117,7 @@ "postcss-import": "^14.1.0", "postcss-loader": "^6.2.1", "postcss-preset-env": "^7.4.3", - "prettier": "^2.7.0", + "prettier": "^2.7.1", "properties-reader": "^2.2.0", "react-is": "^17.0.2", "run-z": "^1.10.1", diff --git a/src/main/webapp/pnpm-lock.yaml b/src/main/webapp/pnpm-lock.yaml index fc0612befb8..4fe9bf3f095 100644 --- a/src/main/webapp/pnpm-lock.yaml +++ b/src/main/webapp/pnpm-lock.yaml @@ -25,20 +25,20 @@ specifiers: '@babel/preset-react': ^7.18.6 '@babel/preset-typescript': ^7.18.6 '@babel/runtime': ^7.20.1 - '@loaders.gl/core': '>=3.2.0 <4.0.0' - '@loaders.gl/gltf': ^3.0.0 - '@loaders.gl/images': '>=3.0.0 <4.0.0' - '@luma.gl/engine': '>=8.4.0 <9.0.0' - '@luma.gl/gltools': '>=8.4.0 <9.0.0' - '@luma.gl/shadertools': ^8.4.0 - '@luma.gl/webgl': '>=8.4.0 <9.0.0' + '@loaders.gl/core': ^3.2.4 + '@loaders.gl/gltf': ^3.2.4 + '@loaders.gl/images': ^3.2.4 + '@luma.gl/engine': ^8.5.14 + '@luma.gl/gltools': ^8.5.14 + '@luma.gl/shadertools': ^8.5.14 + '@luma.gl/webgl': ^8.5.14 '@phylocanvas/phylocanvas.gl': ^1.43.0 '@reduxjs/toolkit': 1.8.5 '@types/lodash': ^4.14.182 '@types/lodash.uniqby': ^4.7.7 '@types/node': ^17.0.39 - '@types/react': ^17.0.39 - '@types/react-dom': ^17.0.11 + '@types/react': ^17.0.45 + '@types/react-dom': ^17.0.17 '@types/react-redux': ^7.1.24 '@types/react-virtualized-auto-sizer': ^1.0.1 '@types/react-window': ^1.8.5 @@ -53,7 +53,7 @@ specifiers: babel-loader: ^8.2.4 babel-plugin-import: ^1.13.5 bootstrap: 3.4.1 - bootstrap-daterangepicker: ^3.0.3 + bootstrap-daterangepicker: ^3.1.0 browserslist: ^4.20.2 clipboard: ^1.7.1 css-loader: ^6.7.1 @@ -68,7 +68,7 @@ specifiers: eslint-plugin-react-hooks: ^4.4.0 expose-loader: ^3.1.0 flexlayout-react: ^0.7.4 - gl-matrix: ^3.0.0 + gl-matrix: ^3.4.3 i: ^0.3.7 immutability-helper: ^3.1.1 jest: ^27.5.1 @@ -84,7 +84,7 @@ specifiers: postcss-loader: ^6.2.1 postcss-nested: ^5.0.6 postcss-preset-env: ^7.4.3 - prettier: ^2.7.0 + prettier: ^2.7.1 process: ^0.11.10 properties-reader: ^2.2.0 qs: ^6.10.3 @@ -3642,7 +3642,7 @@ packages: postcss: ^8.1.0 dependencies: browserslist: 4.20.2 - caniuse-lite: 1.0.30001332 + caniuse-lite: 1.0.30001431 fraction.js: 4.2.0 normalize-range: 0.1.2 picocolors: 1.0.0 @@ -3865,7 +3865,7 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true dependencies: - caniuse-lite: 1.0.30001332 + caniuse-lite: 1.0.30001431 electron-to-chromium: 1.4.107 escalade: 3.1.1 node-releases: 2.0.3 @@ -3877,7 +3877,7 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true dependencies: - caniuse-lite: 1.0.30001430 + caniuse-lite: 1.0.30001431 electron-to-chromium: 1.4.284 node-releases: 2.0.6 update-browserslist-db: 1.0.10_browserslist@4.21.4 @@ -3922,17 +3922,13 @@ packages: resolution: {integrity: sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==} dependencies: browserslist: 4.21.4 - caniuse-lite: 1.0.30001430 + caniuse-lite: 1.0.30001431 lodash.memoize: 4.1.2 lodash.uniq: 4.5.0 dev: true - /caniuse-lite/1.0.30001332: - resolution: {integrity: sha512-10T30NYOEQtN6C11YGg411yebhvpnC6Z102+B95eAsN0oB6KUs01ivE8u+G6FMIRtIrVlYXhL+LUwQ3/hXwDWw==} - dev: true - - /caniuse-lite/1.0.30001430: - resolution: {integrity: sha512-IB1BXTZKPDVPM7cnV4iaKaHxckvdr/3xtctB3f7Hmenx3qYBhGtTZ//7EllK66aKXW98Lx0+7Yr0kxBtIt3tzg==} + /caniuse-lite/1.0.30001431: + resolution: {integrity: sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ==} dev: true /cartocolor/4.0.2: @@ -5367,6 +5363,11 @@ packages: delegate: 3.2.0 dev: false + /graceful-fs/4.2.10: + resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} + dev: true + optional: true + /graceful-fs/4.2.9: resolution: {integrity: sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==} dev: true @@ -6416,11 +6417,11 @@ packages: tslib: 2.3.1 optionalDependencies: errno: 0.1.8 - graceful-fs: 4.2.9 + graceful-fs: 4.2.10 image-size: 0.5.5 make-dir: 2.1.0 mime: 1.6.0 - needle: 2.6.0 + needle: 2.9.1 source-map: 0.6.1 transitivePeerDependencies: - supports-color @@ -6914,8 +6915,8 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true - /needle/2.6.0: - resolution: {integrity: sha512-KKYdza4heMsEfSWD7VPUIz3zX2XDwOyX2d+geb4vrERZMT5RMU6ujjaD+I5Yr54uZxQ2w6XRTAhHBbSCyovZBg==} + /needle/2.9.1: + resolution: {integrity: sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==} engines: {node: '>= 4.4.x'} hasBin: true requiresBuild: true From dda12788acf32f5d5824259d1de89131a7a9d6c0 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Thu, 10 Nov 2022 10:05:59 -0600 Subject: [PATCH 384/655] chore: Removed server side datatables code. --- .../irida/config/web/IridaUIWebConfig.java | 43 ++-- .../irida/ria/utilities/Formats.java | 14 -- .../ria/utilities/components/DataTable.java | 52 ---- .../ria/web/components/SubmissionIds.java | 26 -- .../DataTablesColumnDefinitions.java | 119 ---------- .../datatables/DataTablesExportToFile.java | 108 --------- .../datatables/DataTablesExportTypes.java | 8 - .../datatables/DataTablesExportable.java | 29 --- .../datatables/DataTablesParams.java | 223 ------------------ .../datatables/DataTablesResponse.java | 75 ------ .../datatables/config/DataTablesRequest.java | 29 --- .../config/DataTablesRequestResolver.java | 34 --- .../export/ExportFormatException.java | 10 - .../models/AbstractExportModel.java | 41 ---- .../datatables/models/ProjectSampleModel.java | 117 --------- .../models/datatables/DTProjectSamples.java | 122 ---------- 16 files changed, 15 insertions(+), 1035 deletions(-) delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/Formats.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/components/DataTable.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/SubmissionIds.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesColumnDefinitions.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportToFile.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportTypes.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportable.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesParams.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesResponse.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequest.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequestResolver.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/export/ExportFormatException.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/AbstractExportModel.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/ProjectSampleModel.java delete mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/models/datatables/DTProjectSamples.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java index 3c78831499a..162c9e57d3f 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/config/web/IridaUIWebConfig.java @@ -1,11 +1,16 @@ package ca.corefacility.bioinformatics.irida.config.web; -import java.io.IOException; -import java.nio.file.*; -import java.util.*; - -import javax.servlet.http.HttpServletRequest; - +import ca.corefacility.bioinformatics.irida.config.security.IridaApiSecurityConfig; +import ca.corefacility.bioinformatics.irida.config.services.WebEmailConfig; +import ca.corefacility.bioinformatics.irida.ria.config.AnalyticsHandlerInterceptor; +import ca.corefacility.bioinformatics.irida.ria.config.BreadCrumbInterceptor; +import ca.corefacility.bioinformatics.irida.ria.config.GalaxySessionInterceptor; +import ca.corefacility.bioinformatics.irida.ria.config.UserSecurityInterceptor; +import ca.corefacility.bioinformatics.irida.ria.config.thymeleaf.webpacker.WebpackerDialect; +import ca.corefacility.bioinformatics.irida.ria.web.sessionAttrs.Cart; +import com.github.mxab.thymeleaf.extras.dataattribute.dialect.DataAttributeDialect; +import com.google.common.base.Joiner; +import nz.net.ultraq.thymeleaf.layoutdialect.LayoutDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; @@ -19,7 +24,6 @@ import org.springframework.core.env.Profiles; import org.springframework.http.HttpStatus; import org.springframework.web.context.WebApplicationContext; -import org.springframework.web.method.support.HandlerMethodArgumentResolver; import org.springframework.web.servlet.LocaleResolver; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; @@ -36,19 +40,10 @@ import org.thymeleaf.templateresolver.FileTemplateResolver; import org.thymeleaf.templateresolver.ITemplateResolver; -import ca.corefacility.bioinformatics.irida.config.security.IridaApiSecurityConfig; -import ca.corefacility.bioinformatics.irida.config.services.WebEmailConfig; -import ca.corefacility.bioinformatics.irida.ria.config.AnalyticsHandlerInterceptor; -import ca.corefacility.bioinformatics.irida.ria.config.BreadCrumbInterceptor; -import ca.corefacility.bioinformatics.irida.ria.config.GalaxySessionInterceptor; -import ca.corefacility.bioinformatics.irida.ria.config.UserSecurityInterceptor; -import ca.corefacility.bioinformatics.irida.ria.config.thymeleaf.webpacker.WebpackerDialect; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config.DataTablesRequestResolver; -import ca.corefacility.bioinformatics.irida.ria.web.sessionAttrs.Cart; - -import com.github.mxab.thymeleaf.extras.dataattribute.dialect.DataAttributeDialect; -import com.google.common.base.Joiner; -import nz.net.ultraq.thymeleaf.layoutdialect.LayoutDialect; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.nio.file.*; +import java.util.*; /** */ @@ -257,14 +252,6 @@ public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(userSecurityInterceptor()); } - /** - * {@inheritDoc} - */ - @Override - public void addArgumentResolvers(List argumentResolvers) { - argumentResolvers.add(new DataTablesRequestResolver()); - } - /** * This is to add additional Thymeleaf dialects. * diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/Formats.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/Formats.java deleted file mode 100644 index eccfbc18380..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/Formats.java +++ /dev/null @@ -1,14 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.utilities; - -import java.text.SimpleDateFormat; - -/** - * This class is responsible for holding the formats of different Strings (e.g. Dates) - * - */ -public interface Formats { - /** - * Default format for {@link java.util.Date} in DataTables - */ - public static final SimpleDateFormat DATE = new SimpleDateFormat("dd MMM yyyy"); -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/components/DataTable.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/components/DataTable.java deleted file mode 100644 index f2540b123e4..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/utilities/components/DataTable.java +++ /dev/null @@ -1,52 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.utilities.components; - -import org.springframework.data.domain.Sort; - -/** - * Static string used in the ResponseBody for DataTables calls. - * - */ -public class DataTable { - /** - * Request Parameters - */ - public static final String REQUEST_PARAM_START = "start"; - public static final String REQUEST_PARAM_LENGTH = "length"; - public static final String REQUEST_PARAM_DRAW = "draw"; - public static final String REQUEST_PARAM_SEARCH_VALUE = "search[value]"; - public static final String REQUEST_PARAM_SORT_COLUMN = "order[0][column]"; - public static final String REQUEST_PARAM_SORT_DIRECTION = "order[0][dir]"; - - /** - * Response Parameters - */ - public static final String RESPONSE_PARAM_DRAW = REQUEST_PARAM_DRAW; - public static final String RESPONSE_PARAM_SORT_COLUMN = REQUEST_PARAM_SORT_COLUMN; - public static final String RESPONSE_PARAM_SORT_DIRECTION = REQUEST_PARAM_SORT_DIRECTION; - public static final String RESPONSE_PARAM_DATA = "data"; - public static final String RESPONSE_PARAM_RECORDS_TOTAL = "recordsTotal"; - public static final String RESPONSE_PARAM_RECORDS_FILTERED = "recordsFiltered"; - - /** - * Helper method to get the sort direction for the column. - * - * @param directionString - * Expect either "asc" or "desc" for direction of sort. - * @return {@link org.springframework.data.domain.Sort.Direction} - */ - public static Sort.Direction getSortDirection(String directionString) { - return directionString.equals("asc") ? Sort.Direction.ASC : Sort.Direction.DESC; - } - - /** - * Helper method to get the page number based on the pageSize of the list and - * the current start - * - * @param start The first object at the start of the currently display DataTables row. - * @param length The length of the DataTables page. - * @return Current page number - */ - public static int getPageNumber(int start, int length) { - return start / length; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/SubmissionIds.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/SubmissionIds.java deleted file mode 100644 index 0f5bf8cec52..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/SubmissionIds.java +++ /dev/null @@ -1,26 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components; - -import java.util.ArrayList; -import java.util.List; - -/** - * This class is used to store workflow submission ids in the session. This should be temporary until - * they are stored in the database. Storing a list in a class seems to be handles better by Spring. - * - */ -public class SubmissionIds { - List ids; - - public SubmissionIds() { - ids = new ArrayList<>(); - } - - /** - * add an id to the submission ids - * - * @param id the id to add - */ - public void addId(Long id) { - ids.add(id); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesColumnDefinitions.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesColumnDefinitions.java deleted file mode 100644 index 9303b433fe6..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesColumnDefinitions.java +++ /dev/null @@ -1,119 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables; - -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config.DataTablesRequest; -import com.google.common.base.Strings; - -import javax.servlet.http.HttpServletRequest; - -/** - * Responsible for extracting and representing DataTables Columns - * - * 
    - *  columns[n][data]            - Column's data source (https://datatables.net/reference/option/columns.data)
- *  columns[n][name]            - Column's name (https://datatables.net/reference/option/columns.name)
- *  columns[n][searchable]      - Flag to indicate if this column is globally searchable (true) or not (false).
- *  columns[n][orderable]       - Flag to indicate if this column is orderable (true) or not (false).
- *  columns[n][search][value]   - Search value to apply to this specific column.
- *  columns[n][search][regex]   - Flag to indicate if the search term for this column should be treated as
- *                                regular expression (true) or not (false).
- *
    - * - * @see Server-side - * processing - */ -public class DataTablesColumnDefinitions { - private String name; - private String data; - private boolean orderable; - private boolean searchable; - private String searchValue; - - private DataTablesColumnDefinitions(String name, String data, boolean searchable, boolean orderable, - String searchValue) { - this.name = name; - this.data = data; - this.searchable = searchable; - this.orderable = orderable; - this.searchValue = searchValue; - } - - /** - * Static initializer. Parses {@link HttpServletRequest} to get the - * DataTables parameters for the Column. - * - * @param index - * {@link Integer} index for the current column. - * @param request - * {@link HttpServletRequest} current server request, containing - * a {@link DataTablesRequest} - * - * @return {@link DataTablesColumnDefinitions} definition for the column at - * the index. - */ - static DataTablesColumnDefinitions createColumnDefinition(Integer index, HttpServletRequest request) { - String prefix = "columns[" + index + "]"; - String name = request.getParameter(prefix + "[name]"); - if (Strings.isNullOrEmpty(name)) { - name = request.getParameter(prefix + "[data]"); - } - String data = request.getParameter(prefix + "[data]"); - boolean searchable = Boolean.parseBoolean(request.getParameter(prefix + "[searchable]")); - boolean orderable = Boolean.parseBoolean(request.getParameter(prefix + "[orderable]")); - String searchValue = request.getParameter(prefix + "[search][value]"); - return new DataTablesColumnDefinitions(name, data, searchable, orderable, searchValue); - } - - /** - * Column Name - * - * @return {@link String} - * @see columns.name - */ - public String getColumnName() { - return name; - } - - /** - * Column Data - * - * @return {@link String} - * @see columns.data - */ - public String getData() { - return data; - } - - /** - * Column Orderable: end user's ability to order this column. - * - * @return {@link Boolean} true if the column is orderable. - * @see columns.orderable - */ - public boolean isOrderable() { - return orderable; - } - - /** - * Column searchable Defined if DataTables should include this column in the - * filterable data in the table. - * - * @return {@link Boolean} true if the column is searchable - * @see solumns.searchable - */ - public boolean isSearchable() { - return searchable; - } - - /** - * Get the value searched for in this column - * - * @return {@link String} - */ - public String getSearchValue() { - return searchValue; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportToFile.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportToFile.java deleted file mode 100644 index f68303ee67f..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportToFile.java +++ /dev/null @@ -1,108 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.ArrayList; -import java.util.List; - -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.xssf.usermodel.XSSFSheet; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; - -/** - * Used to export datatables to either excel or csv formatted files. - */ -public class DataTablesExportToFile { - - /** - * Write data within datatable to an excel formatted file. - * - * @param type {@link DataTablesExportTypes} type of file to create (either excel or csv) - * @param response {@link HttpServletResponse} - * @param filename {@link String} name of the file to download. - * @param models Data to download in the table - * @param headers for the table - * @throws IOException thrown if file cannot be written - */ - public static void writeFile(DataTablesExportTypes type, HttpServletResponse response, String filename, - List models, List headers) throws IOException { - if (type.equals(DataTablesExportTypes.excel)) { - writeToExcel(response, filename, models, headers); - } else if (type.equals(DataTablesExportTypes.csv)) { - writeToCSV(response, filename, models, headers); - } else { - throw new IllegalArgumentException("Trying to export and unknown table format: " + type); - } - } - - /** - * Write data within datatable to an excel formatted file. - * - * @param response {@link HttpServletResponse} - * @param filename {@link String} name of the file to download. - * @param models Data to download in the table - * @param headers for the table - * @throws IOException thrown if file cannot be written - */ - private static void writeToExcel(HttpServletResponse response, String filename, - List models, List headers) throws IOException { - XSSFWorkbook workbook = new XSSFWorkbook(); - XSSFSheet sheet = workbook.createSheet(); - - // Create the header row - Row row = sheet.createRow(0); - int cellNum = 0; - for (String header : headers) { - Cell cell = row.createCell(cellNum++); - cell.setCellValue(header); - } - - // Add the data to the workbook - int rowNum = 1; - for (DataTablesExportable model : models) { - row = sheet.createRow(rowNum++); - int cellCount = 0; - for (String content : model.getExportableTableRow()) { - Cell cell = row.createCell(cellCount++); - cell.setCellValue(content); - } - } - - response.setContentType("application/vnd.ms-excel"); - response.setHeader("Content-disposition", "attachment; filename=" + filename + ".xlsx"); - workbook.write(response.getOutputStream()); - - workbook.close(); - } - - /** - * Write data within datatable to a csv formatted file. - * - * @param response {@link HttpServletResponse} - * @param filename {@link String} name of the file to download. - * @param models Data to download in the table - * @param headers for the table - * @throws IOException thrown if file cannot be written - */ - private static void writeToCSV(HttpServletResponse response, String filename, - List models, List headers) throws IOException { - List results = new ArrayList<>(); - results.add(headers.toArray(new String[0])); - for (DataTablesExportable model : models) { - results.add(model.getExportableTableRow().toArray(new String[0])); - } - - response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".csv\""); - response.setContentType("text/csv"); - OutputStreamWriter outputStreamWriter = new OutputStreamWriter(response.getOutputStream()); - CSVPrinter printer = CSVFormat.DEFAULT.print(outputStreamWriter); - printer.printRecords(results); - printer.flush(); - outputStreamWriter.close(); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportTypes.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportTypes.java deleted file mode 100644 index 3477e2c2fe1..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportTypes.java +++ /dev/null @@ -1,8 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables; - -/** - * This class represents the types of files that can be exported from a datatable. - */ -public enum DataTablesExportTypes { - excel, csv -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportable.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportable.java deleted file mode 100644 index 805c21d45f4..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesExportable.java +++ /dev/null @@ -1,29 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables; - -import java.util.List; -import java.util.Locale; - -import org.springframework.context.MessageSource; - -/** - * This interface is responsible for enforcing that classes implementing - * {@link ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models.DataTablesResponseModel} and - * need to be exportable will be able to explicitly state which attributes and in which order they need to be exported. - */ -public interface DataTablesExportable { - /** - * Convert the attribute of the class into a ordered list. - * - * @return List of values in order for the specific datatable. - */ - List getExportableTableRow(); - - /** - * Get an ordered list of internationalized table headers for the datatable. - * - * @param messageSource {@link MessageSource} - * @param locale {@link Locale} for the current user. - * @return List of table headers. - */ - List getExportableTableHeaders(MessageSource messageSource, Locale locale); -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesParams.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesParams.java deleted file mode 100644 index 697873aba66..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesParams.java +++ /dev/null @@ -1,223 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables; - -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.servlet.http.HttpServletRequest; - -import org.springframework.data.domain.Sort; - -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config.DataTablesRequest; - -import com.google.common.base.Strings; - -/** - * Represents all parameters within a DataTables request. - * 
    - * draw                  - Draw counter. This is used by DataTables to ensure that the Ajax returns
- *                         from server-side processing requests are drawn in sequence by DataTables.
- * start                 - Paging first record indicator. This is the start point in the current
- *                         data set (0 index based - i.e. 0 is the first record).
- * length                - Number of records that the table can display in the current draw.
- * search[value]         - Global search value. To be applied to all columns which have
- *                         searchable as true.
- * search[regex]         - true if the global filter should be treated as a regular expression for
- *                         advanced searching, false otherwise.
- *
    - * - * @see Server Side - */ -public class DataTablesParams { - // This is used to match DataTables columns parameter - // e.g. columns[1][name] - // - matching at 1 ==> column number - // - matching at 2 ==> which data item (name, seachable, orderable, data) - private static Pattern columnsPattern = Pattern.compile("columns\\[([0-9]*)?\\]\\[([a-z]*)?\\]"); - - private Integer start; - private Integer length; - private Integer draw; - private String searchValue; - private Sort sort; - private Map searchMap; - - public DataTablesParams() { - } - - public DataTablesParams(Integer start, Integer length, Integer draw, String searchValue, Sort sort, Map searchMap) { - this.start = start; - this.length = length; - this.draw = draw; - this.searchValue = searchValue; - this.sort = sort; - this.searchMap = searchMap; - } - - /** - * Static initializer to get the params for all {@link DataTablesRequest} - * - * @param request - * {@link HttpServletRequest} request containing {@link DataTablesParams} - * @return {@link DataTablesParams} - */ - public static DataTablesParams parseDataTablesParams(HttpServletRequest request) { - Integer start = Integer.valueOf(request.getParameter("start")); - Integer length = Integer.valueOf(request.getParameter("length")); - Integer draw = Integer.valueOf(request.getParameter("draw")); - String searchValue = request.getParameter("search[value]"); - List dataTablesColumnDefinitions = getColumnDefinitions(request); - - // Get any column specific search data - Map searchMap = new HashMap<>(); - for (DataTablesColumnDefinitions def : dataTablesColumnDefinitions) { - if (!Strings.isNullOrEmpty(def.getSearchValue())) { - searchMap.put(def.getColumnName(), def.getSearchValue()); - } - } - - Sort sort = getColumnSortData(request, dataTablesColumnDefinitions); - return new DataTablesParams(start, length, draw, searchValue, sort, searchMap); - } - - /** - * Get the DataTables {@link Sort} information from the {@link HttpServletRequest} - * - * @param request - * {@link HttpServletRequest} current {@link DataTablesParams} request. - * @param columnDefinitions - * {@link List} of {@link DataTablesColumnDefinitions} - * @return {@link Sort} of containing all {@link DataTablesColumnDefinitions} sort information. - */ - private static Sort getColumnSortData(HttpServletRequest request, - List columnDefinitions) { - String orderColumnPattern = "order[0][column]"; - String orderDirectionPattern = "order[0][dir]"; - int columnNumber = Integer.parseInt(request.getParameter(orderColumnPattern)); - String directionString = request.getParameter(orderDirectionPattern); - - List sortOrder = new ArrayList<>(); - sortOrder.add(new Sort.Order(getSortDirectionFromString(directionString), - columnDefinitions.get(columnNumber).getColumnName())); - - boolean notDone = true; - for (int i = 1; notDone; i++) { - directionString = request.getParameter(orderDirectionPattern.replace("0", i + "")); - if (!Strings.isNullOrEmpty(directionString)) { - columnNumber = Integer.parseInt(request.getParameter(orderColumnPattern.replace("0", i + ""))); - sortOrder.add(new Sort.Order(getSortDirectionFromString(directionString), - columnDefinitions.get(columnNumber).getColumnName())); - } else { - notDone = false; - } - } - - return Sort.by(sortOrder); - } - - /** - * Convert string sort direction information into a {@link Sort.Direction} - * - * @param direction - * {@link String} value of the sort. - * Expect either "asc" ord "desc" - * @return {@link Sort.Direction} - */ - private static Sort.Direction getSortDirectionFromString(String direction) { - return direction.equals("asc") ? Sort.Direction.ASC : Sort.Direction.DESC; - } - - /** - * Extract the DataTables column definitions from the {@link HttpServletRequest} - * - * @param request - * {@link HttpServletRequest} current {@link DataTablesParams} request. - * @return {@link List} of {@link DataTablesColumnDefinitions}p - */ - private static List getColumnDefinitions(HttpServletRequest request) { - Enumeration parameterNames = request.getParameterNames(); - Map foundMap = new HashMap<>(); - - // Iterate over all the named parameters. If it matches what a column parameter should look like - // Create the column definition. - while (parameterNames.hasMoreElements()) { - String param = parameterNames.nextElement(); - Matcher matcher = columnsPattern.matcher(param); - while (matcher.find()) { - // Group 0 = entire - // Group 1 = column number - Integer column = Integer.valueOf(matcher.group(1)); - if (!foundMap.containsKey(column)) { - foundMap.put(column, DataTablesColumnDefinitions.createColumnDefinition(column, request)); - } - } - } - - // Keys should always be 0 - n - List dataTablesColumnDefinitions = new ArrayList<>(); - for (int i = 0; i < foundMap.keySet().size(); i++) { - dataTablesColumnDefinitions.add(foundMap.get(i)); - } - - return dataTablesColumnDefinitions; - } - - /** - * Get the draw counter. This is used by DataTables to ensure that the Ajax returns from server-side - * processing requests are drawn in sequence by DataTables (Ajax requests are asynchronous and thus can return - * out of sequence). This is used as part of the draw return parameter. - * - * @return {@link Integer} draw counter - */ - public int getDraw() { - return draw; - } - - /** - * Number of records that the table can display in the current draw. - * - * @return {@link Integer} number of records in the current page. - */ - public int getLength() { - return length; - } - - /** - * Get the currently viewed page in the DataTable. - * - * @return {@link Integer} Current page - */ - public int getCurrentPage() { - return (int) Math.floor(start / length); - } - - /** - * Get the value from the global search. - * - * @return {@link String} search value - */ - public String getSearchValue() { - return searchValue; - } - - /** - * Get the current table sort properties. - * - * @return {@link Sort} current table sort. - */ - public Sort getSort() { - return sort; - } - - /** - * Get all the searches on this columns in this table - * @return {@link Map} - */ - public Map getSearchMap() { - return searchMap; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesResponse.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesResponse.java deleted file mode 100644 index b6c80f4badd..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/DataTablesResponse.java +++ /dev/null @@ -1,75 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables; - -import java.util.List; - -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models.DataTablesResponseModel; -import com.google.common.collect.Lists; -import org.springframework.data.domain.Page; - -/** - * This is returned to client in an {@link javax.servlet.http.HttpServletResponse} in the format the can be consumed - * by jQuery DataTables. - * - * @see DataTables - */ -public class DataTablesResponse { - private DataTablesParams dataTablesParams; - private Long recordsTotal; - private Long recordsFiltered; - private List data; - - public DataTablesResponse(DataTablesParams dataTablesParams, Page page, List data) { - this.dataTablesParams = dataTablesParams; - this.recordsTotal = page.getTotalElements(); - this.recordsFiltered = page.getTotalElements(); - this.data = data; - } - - public DataTablesResponse(DataTablesParams dataTablesParams, long recordsTotal, - List data) { - this.dataTablesParams = dataTablesParams; - this.recordsTotal = recordsTotal; - this.recordsFiltered = recordsTotal; - this.data = data; - } - - /** - * Get the draw counter. This is used by DataTables to ensure that the Ajax returns from server-side processing - * requests are drawn in sequence by DataTables. Extracted directly from the initial - * {@code org.apache.http.HttpRequest} - * - * @return {@link Integer} - */ - public int getDraw() { - return dataTablesParams.getDraw(); - } - - /** - * Total records, before filtering (i.e. the total number of records in the database) - * - * @return {@link Long} - */ - public long getRecordsTotal() { - return recordsTotal; - } - - /** - * Total records, after filtering (i.e. the total number of records after filtering has been applied - * - not just the number of records being returned for this page of data). - * - * @return {@link Long} - */ - public long getRecordsFiltered() { - return recordsFiltered; - } - - /** - * The data to be displayed in the table. This is an array of data source objects, one for each row, - * which will be used by DataTables. - * - * @return {@link List} - */ - public List getData() { - return Lists.newArrayList(data); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequest.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequest.java deleted file mode 100644 index c7d0646cb2e..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequest.java +++ /dev/null @@ -1,29 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import javax.servlet.http.HttpServletRequest; - -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesColumnDefinitions; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesParams; - -/** - * Annotation for {@link DataTablesParams} - * This is used to handle an ajax request from a client for a DataTables - * response. This is used to intercept the {@link HttpServletRequest} and capture - * server side DataTables parameters. - * - * This dependent on: {@link DataTablesRequestResolver}, {@link DataTablesParams}, {@link DataTablesColumnDefinitions}. - * - * Intended usage: - * {@code public DataTablesResponse getDataTablesResponse(@DataTablesRequest DataTablesParams params) { ... }} - */ -@Target(ElementType.PARAMETER) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface DataTablesRequest { -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequestResolver.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequestResolver.java deleted file mode 100644 index e6cf4708fa7..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/config/DataTablesRequestResolver.java +++ /dev/null @@ -1,34 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables.config; - -import javax.servlet.http.HttpServletRequest; - -import org.springframework.core.MethodParameter; -import org.springframework.web.bind.support.WebDataBinderFactory; -import org.springframework.web.context.request.NativeWebRequest; -import org.springframework.web.method.support.HandlerMethodArgumentResolver; -import org.springframework.web.method.support.ModelAndViewContainer; - -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesParams; - -/** - * Request resolver for {@link DataTablesParams} - */ -public class DataTablesRequestResolver implements HandlerMethodArgumentResolver { - @Override - public boolean supportsParameter(MethodParameter parameter) { - DataTablesRequest dataTablesRequest = parameter.getParameterAnnotation(DataTablesRequest.class); - if (dataTablesRequest != null) { - if (DataTablesParams.class.isAssignableFrom(parameter.getParameterType())) { - return true; - } - } - return false; - } - - @Override - public Object resolveArgument(MethodParameter parameter, ModelAndViewContainer mavContainer, - NativeWebRequest webRequest, WebDataBinderFactory binderFactory) throws Exception { - HttpServletRequest servletRequest = (HttpServletRequest) webRequest.getNativeRequest(); - return DataTablesParams.parseDataTablesParams(servletRequest); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/export/ExportFormatException.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/export/ExportFormatException.java deleted file mode 100644 index 56b387d7bf5..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/export/ExportFormatException.java +++ /dev/null @@ -1,10 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables.export; - -/** - * {@link Exception} thrown when attempting to export a table with an unknown format. - */ -public class ExportFormatException extends RuntimeException { - public ExportFormatException(String format) { - super("Attempting to export via an unknown format [" + format + "]"); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/AbstractExportModel.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/AbstractExportModel.java deleted file mode 100644 index 1b0b902db8d..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/AbstractExportModel.java +++ /dev/null @@ -1,41 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models; - -import java.util.Date; - -import com.google.common.base.Strings; - -/** - * Parent class of all export model to hold utility functions. - */ -public abstract class AbstractExportModel { - - /** - * Check if the ID of the object is null. If null, return empty string. - * - * @param id the ID to check - * @return the value or empty string - */ - protected String checkNullId(Long id) { - return id != null ? Long.toString(id) : ""; - } - - /** - * Check if a string is null or empty. If null, return empty string. - * - * @param item the item to check - * @return the value or empty string - */ - protected String checkNullStrings(String item) { - return Strings.isNullOrEmpty(item) ? "" : item; - } - - /** - * Check if a date is null. If null, return empty string. - * - * @param date the date to check. - * @return the date as a string or an empty string. - */ - protected String checkNullDate(Date date) { - return date != null ? date.toString() : ""; - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/ProjectSampleModel.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/ProjectSampleModel.java deleted file mode 100644 index 9fcb3dd52e3..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/components/datatables/models/ProjectSampleModel.java +++ /dev/null @@ -1,117 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models; - -import java.util.List; - -import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; -import ca.corefacility.bioinformatics.irida.model.project.Project; -import ca.corefacility.bioinformatics.irida.model.sample.QCEntry; -import ca.corefacility.bioinformatics.irida.model.sample.Sample; - -import com.google.common.collect.ImmutableList; - -/** - * Used for exporting the project samples table. Prevents undefined from displaying in the table, - * and adds the project name. - */ -public class ProjectSampleModel extends AbstractExportModel { - /** - * Attributes on the {@link Sample} - */ - public static List attributes = ImmutableList.of( - "id", - "sampleName", - "projectName", - "createdDate", - "modifiedDate", - "description", - "organism", - "isolate", - "strain", - "collectedBy", - "collectionDate", - "geographicLocationName", - "isolationSource", - "latitude", - "longitude", - "owner" - ); - private Project project; - private Sample sample; - private List qcEntries; - private ProjectSampleJoin join; - - public ProjectSampleModel(ProjectSampleJoin psj, List qcEntries) { - this.project = psj.getSubject(); - this.sample = psj.getObject(); - this.qcEntries = qcEntries; - this.join = psj; - } - - public String getId() { - return checkNullId(sample.getId()); - } - - public String getSampleName() { - return checkNullStrings(sample.getSampleName()); - } - - public Long getProjectId() { - return project.getId(); - } - - public String getProjectName() { - return checkNullStrings(project.getName()); - } - - public String getCreatedDate() { - return checkNullDate(sample.getCreatedDate()); - } - - public String getModifiedDate() { - return checkNullDate(sample.getModifiedDate()); - } - - public String getDescription() { - return checkNullStrings(sample.getDescription()); - } - - public String getOrganism() { - return checkNullStrings(sample.getOrganism()); - } - - public String getIsolate() { - return checkNullStrings(sample.getIsolate()); - } - - public String getCollectedBy() { - return checkNullStrings(sample.getCollectedBy()); - } - - public String getCollectionDate() { - return checkNullDate(sample.getCollectionDate()); - } - - public String getGeographicLocationName() { - return checkNullStrings(sample.getGeographicLocationName()); - } - - public String getIsolationSource() { - return checkNullStrings(sample.getIsolationSource()); - } - - public String getLatitude() { - return checkNullStrings(sample.getLatitude()); - } - - public String getLongitude() { - return checkNullStrings(sample.getLongitude()); - } - - public List getQcEntries(){ - return qcEntries; - } - - public boolean isOwner(){ - return join.isOwner(); - } -} diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/models/datatables/DTProjectSamples.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/models/datatables/DTProjectSamples.java deleted file mode 100644 index 1a6cdb9c9f4..00000000000 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/models/datatables/DTProjectSamples.java +++ /dev/null @@ -1,122 +0,0 @@ -package ca.corefacility.bioinformatics.irida.ria.web.models.datatables; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.*; - -import org.springframework.context.MessageSource; - -import ca.corefacility.bioinformatics.irida.model.joins.impl.ProjectSampleJoin; -import ca.corefacility.bioinformatics.irida.model.project.Project; -import ca.corefacility.bioinformatics.irida.model.sample.Sample; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.DataTablesExportable; -import ca.corefacility.bioinformatics.irida.ria.web.components.datatables.models.DataTablesResponseModel; - -/** - * DataTables response object for {@link ProjectSampleJoin} - */ -public class DTProjectSamples implements DataTablesResponseModel, DataTablesExportable { - private final String dataPattern = "MMM dd, yyyy"; - private final DateFormat dateFormatter = new SimpleDateFormat(dataPattern); - - private Long id; - private Long projectId; - private String sampleName; - private String organism; - private String projectName; - private Date createdDate; - private Date modifiedDate; - private List qcEntries; - private Double coverage; - private boolean owner; - - public DTProjectSamples(ProjectSampleJoin projectSampleJoin, List qcEntries, Double coverage) { - Project project = projectSampleJoin.getSubject(); - Sample sample = projectSampleJoin.getObject(); - - this.id = sample.getId(); - this.sampleName = sample.getSampleName(); - this.organism = sample.getOrganism(); - this.projectName = project.getName(); - this.projectId = project.getId(); - this.createdDate = sample.getCreatedDate(); - this.modifiedDate = sample.getModifiedDate(); - this.qcEntries = qcEntries; - this.coverage = coverage; - this.owner = projectSampleJoin.isOwner(); - } - - public Long getId() { - return this.id; - } - - public String getSampleName() { - return sampleName; - } - - public String getOrganism() { - return organism; - } - - public Long getProjectId() { - return projectId; - } - - public String getProjectName() { - return projectName; - } - - public Date getCreatedDate() { - return createdDate; - } - - public Date getModifiedDate() { - return modifiedDate; - } - - public List getQcEntries() { - return qcEntries; - } - - public Double getCoverage() { - return coverage; - } - - public boolean isOwner(){ - return owner; - } - - /** - * {@inheritDoc} - */ - @Override - public List getExportableTableRow() { - List data = new ArrayList<>(); - data.add(String.valueOf(this.getId())); - data.add(this.getSampleName()); - data.add(this.getOrganism()); - data.add(String.valueOf(this.getProjectId())); - data.add(this.getProjectName()); - data.add(this.getCreatedDate() != null ? dateFormatter.format(this.getCreatedDate()) : ""); - data.add(this.getModifiedDate() != null ? dateFormatter.format(this.getModifiedDate()) : ""); - data.add(this.getCoverage() != null ? String.valueOf(this.getCoverage()) : ""); - return data; - } - - /** - * {@inheritDoc} - */ - @Override - public List getExportableTableHeaders(MessageSource messageSource, Locale locale) { - List headers = new ArrayList<>(); - headers.add(messageSource.getMessage("iridaThing.id", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.name", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.organism", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.project-id", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.project", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.created", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.modified", new Object[] {}, locale)); - headers.add(messageSource.getMessage("project.samples.table.coverage", new Object[] {}, locale)); - return headers; - } -} From cb67fb113789257b6bca9ceae9181b210511e4cd Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 10 Nov 2022 10:49:14 -0600 Subject: [PATCH 385/655] changing 'Map Headers' to 'Map Columns' --- src/main/resources/i18n/messages.properties | 17 +-- ...tsx => SampleMetadataImportMapColumns.tsx} | 108 +++++++++++------- .../SampleMetadataImportUploadFile.tsx | 2 +- .../projects/samples-metadata-import/index.js | 6 +- .../ProjectSampleMetadataImportPage.java | 2 +- 5 files changed, 83 insertions(+), 52 deletions(-) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportMapHeaders.tsx => SampleMetadataImportMapColumns.tsx} (62%) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index f93b974822f..f8aa6a582c9 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -1985,7 +1985,7 @@ SampleMetadataImportWizard.title=Sample Metadata Uploader SampleMetadataImportWizard.intro=A tool you can use to upload excel or csv files containing metadata for samples in this project. SampleMetadataImportSteps.step1=Upload File -SampleMetadataImportSteps.step2=Map Headers +SampleMetadataImportSteps.step2=Map Columns SampleMetadataImportSteps.step3=Review Data SampleMetadataImportSteps.step4=Complete @@ -1994,13 +1994,14 @@ SampleMetadataImportUploadFile.dropzone=Click or drop Excel/CSV file containing SampleMetadataImportUploadFile.success={0} was uploaded successfully. SampleMetadataImportUploadFile.error={0} failed to upload. -SampleMetadataImportMapHeaders.description=Select which column maps to the sample name. -SampleMetadataImportMapHeaders.button.back=Upload a new file -SampleMetadataImportMapHeaders.button.next=Preview the data -SampleMetadataImportMapHeaders.table.empty=Waiting on the selection of the sample name column. -SampleMetadataImportMapHeaders.table.header=Header -SampleMetadataImportMapHeaders.table.restriction=Restriction -SampleMetadataImportMapHeaders.table.title=This list of headers from the import file will be created into metadata fields. +SampleMetadataImportMapColumns.form.sampleNameColumn=Select Sample Name Column +SampleMetadataImportMapColumns.form.metadataColumns=Review Parsed Metadata Columns +SampleMetadataImportMapColumns.button.back=Upload a new file +SampleMetadataImportMapColumns.button.next=Review the data +SampleMetadataImportMapColumns.table.empty=Waiting on the selection of the sample name column. +SampleMetadataImportMapColumns.table.header=Header +SampleMetadataImportMapColumns.table.existingRestriction=Existing Restriction +SampleMetadataImportMapColumns.table.targetRestriction=Target Restriction SampleMetadataImportReview.description=Review the metadata to be uploaded. SampleMetadataImportReview.button.back=Select a different column diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx similarity index 62% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx index 5025929a24a..d945b67ed1f 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapHeaders.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx @@ -1,6 +1,15 @@ import React, { useMemo } from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { Button, Empty, Radio, Select, Table, Typography } from "antd"; +import { + Button, + Empty, + Form, + Radio, + Select, + Space, + Table, + Typography, +} from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { IconArrowLeft, @@ -28,7 +37,7 @@ const { Text } = Typography; * @returns {*} * @constructor */ -export function SampleMetadataImportMapHeaders(): JSX.Element { +export function SampleMetadataImportMapColumns(): JSX.Element { const { projectId } = useParams<{ projectId: string }>(); const navigate: NavigateFunction = useNavigate(); const [loading, setLoading] = React.useState(false); @@ -83,25 +92,36 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { [updatedSampleNameColumn, updatedHeaders] ); + const restrictionOptions = restrictions.map(({ label, value }) => ( + + {label} + + )); + const columns = [ { - title: i18n("SampleMetadataImportMapHeaders.table.header"), + title: i18n("SampleMetadataImportMapColumns.table.header"), dataIndex: "name", }, { - title: i18n("SampleMetadataImportMapHeaders.table.restriction"), + title: i18n("SampleMetadataImportMapColumns.table.existingRestriction"), + dataIndex: "existingRestriction", + }, + { + title: i18n("SampleMetadataImportMapColumns.table.targetRestriction"), dataIndex: "restriction", render(id: number, item: MetadataHeaderItem) { return ( onRestrictionChange({ ...item }, value) } - optionType="button" - /> + > + {/* TODO: Change space to compact mode after antd update */} + {restrictionOptions} + ); }, }, @@ -109,43 +129,53 @@ export function SampleMetadataImportMapHeaders(): JSX.Element { return ( - {i18n("SampleMetadataImportMapHeaders.description")} - -
    i18n("SampleMetadataImportMapHeaders.table.title")} - className="t-metadata-uploader-headers-table" - rowKey={(row) => row.rowKey} - columns={columns} - dataSource={dataSource} - pagination={false} - scroll={{ y: 600 }} - locale={{ - emptyText: ( - - ), - }} - /> + + + + + +
    row.rowKey} + columns={columns} + dataSource={dataSource} + pagination={false} + scroll={{ y: 600 }} + locale={{ + emptyText: ( + + ), + }} + /> + +
    diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx index a08d52541d8..87c48fe45a9 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx @@ -79,7 +79,7 @@ export function SampleMetadataImportUploadFile(): JSX.Element { info.file.name ), }); - navigate(`/${projectId}/sample-metadata/upload/headers`); + navigate(`/${projectId}/sample-metadata/upload/columns`); } else if (status === "error") { setLoading(false); setStatus("error"); diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js index a2a170c0ecc..d6649bfeb51 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js @@ -3,7 +3,7 @@ import { BrowserRouter, Route, Routes } from "react-router-dom"; import { Provider } from "react-redux"; import { render } from "react-dom"; import { SampleMetadataImportComplete } from "./components/SampleMetadataImportComplete"; -import { SampleMetadataImportMapHeaders } from "./components/SampleMetadataImportMapHeaders"; +import { SampleMetadataImportMapColumns } from "./components/SampleMetadataImportMapColumns"; import { SampleMetadataImportReview } from "./components/SampleMetadataImportReview"; import { SampleMetadataImportUploadFile } from "./components/SampleMetadataImportUploadFile"; import { setBaseUrl } from "../../../utilities/url-utilities"; @@ -23,8 +23,8 @@ render( element={} /> } + path="/:projectId/sample-metadata/upload/columns" + element={} /> Date: Thu, 10 Nov 2022 11:18:42 -0600 Subject: [PATCH 386/655] fixing formatting --- .../SampleMetadataImportMapColumns.tsx | 48 +++++++------------ 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx index d945b67ed1f..45b013b8bfe 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportMapColumns.tsx @@ -1,15 +1,6 @@ import React, { useMemo } from "react"; import { useNavigate, useParams } from "react-router-dom"; -import { - Button, - Empty, - Form, - Radio, - Select, - Space, - Table, - Typography, -} from "antd"; +import { Button, Empty, Radio, Select, Space, Table, Typography } from "antd"; import { SampleMetadataImportWizard } from "./SampleMetadataImportWizard"; import { IconArrowLeft, @@ -29,8 +20,6 @@ import { } from "../redux/store"; import { getMetadataRestrictions } from "../../../../apis/metadata/field"; -const { Text } = Typography; - /** * React component that displays Step #2 of the Sample Metadata Uploader. * This page is where the user selects the sample name column. @@ -92,12 +81,6 @@ export function SampleMetadataImportMapColumns(): JSX.Element { [updatedSampleNameColumn, updatedHeaders] ); - const restrictionOptions = restrictions.map(({ label, value }) => ( - - {label} - - )); - const columns = [ { title: i18n("SampleMetadataImportMapColumns.table.header"), @@ -113,15 +96,14 @@ export function SampleMetadataImportMapColumns(): JSX.Element { render(id: number, item: MetadataHeaderItem) { return ( onRestrictionChange({ ...item }, value) } - > - {/* TODO: Change space to compact mode after antd update */} - {restrictionOptions} - + optionType="button" + /> ); }, }, @@ -129,10 +111,11 @@ export function SampleMetadataImportMapColumns(): JSX.Element { return ( -
    - + + + + {i18n("SampleMetadataImportMapColumns.form.sampleNameColumn")} + - - + + + + {i18n("SampleMetadataImportMapColumns.form.metadataColumns")} +
    row.rowKey} @@ -167,8 +151,8 @@ export function SampleMetadataImportMapColumns(): JSX.Element { ), }} /> - - + +
    + + ); + }, }, { key: `organism`, From abc797623efa829179947822e507cde13ab87f87 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Tue, 15 Nov 2022 04:59:40 -0600 Subject: [PATCH 392/655] test: Working on getting user search tests passing --- .../resources/js/pages/search/SearchCount.tsx | 1 + .../js/pages/search/SearchLayout.tsx | 3 + .../js/pages/search/SearchProjectsTable.tsx | 1 + .../ria/integration/SearchResultPageIT.java | 77 ++++++++----------- .../integration/pages/SearchResultPage.java | 77 ++++++++++++++++++- .../irida/ria/web/ProjectsPageIT.xml | 3 + 6 files changed, 114 insertions(+), 48 deletions(-) diff --git a/src/main/webapp/resources/js/pages/search/SearchCount.tsx b/src/main/webapp/resources/js/pages/search/SearchCount.tsx index 6d4beae8a79..55085da30f2 100644 --- a/src/main/webapp/resources/js/pages/search/SearchCount.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchCount.tsx @@ -13,6 +13,7 @@ export default function SearchCount({ count }: { count: number }) { overflowCount={1000} style={{ backgroundColor: `var(--grey-7)` }} count={count} + className="t-count-badge" /> ); } diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index 2e1209b5ae5..6ceac04d728 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -226,6 +226,7 @@ export default function SearchLayout() { style={{ width: 120 }} defaultValue="global" onChange={(value) => setGlobal(value === "global")} + className="t-admin-search-type" > PERSONAL GLOBAL @@ -256,6 +257,7 @@ export default function SearchLayout() { debouncedSetSearchParams({ query: e.target.value }) @@ -274,6 +276,7 @@ export default function SearchLayout() { }} selectedKeys={[type]} onClick={(e) => setType(e.key as SearchType)} + className="t-search-nav" > {menuItems.map((item) => ( {item.label} diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index 0e6268bb109..20122cbe3da 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -76,6 +76,7 @@ export default function SearchProjectsTable({ projects?.total ? getPaginationOptions(projects.total) : undefined } onChange={handleTableChange} + className="t-search-projects" /> ); } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/SearchResultPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/SearchResultPageIT.java index 55e4b63df8c..b04b8f5b236 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/SearchResultPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/SearchResultPageIT.java @@ -5,7 +5,7 @@ import com.github.springtestdbunit.annotation.DatabaseSetup; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Test class for global search @@ -17,52 +17,39 @@ public class SearchResultPageIT extends AbstractIridaUIITChromeDriver { public void testGlobalSearchAsAdmin() { LoginPage.loginAsAdmin(driver()); SearchResultPage searchResultPage = new SearchResultPage(driver()); - final String query = "samp"; + final String query = "2"; searchResultPage.enterSearchQueryInNavBar(query); assertTrue(driver().getCurrentUrl().contains("search?query=" + query)); + assertEquals(query, searchResultPage.getSearchInputQuery()); + int numRows = searchResultPage.getTotalNumberOfProjectsInTable(); + assertEquals(2, numRows, "Expected number of projects"); + assertEquals(numRows, searchResultPage.getTotalNumberOfProjectsByBadge()); + assertEquals(1, searchResultPage.getTotalNumberOfSamplesByBadge(), "Expected total number of samples"); + + assertTrue(searchResultPage.isAdminSearchTypeDisplayed(), "Admins should have the ability to switch between global and personal projects"); + searchResultPage.selectAdminPersonalProject(); + assertEquals(0, searchResultPage.getTotalNumberOfProjectsByBadge(), "Admin has no projects containing the search term " + query); + assertEquals(0, searchResultPage.getTotalNumberOfSamplesByBadge(), "Admin has no samples containing the search term " + query); + + String newQuery = "1"; + searchResultPage.enterNewSearchQuery(newQuery); + assertEquals(1, searchResultPage.getTotalNumberOfProjectsByBadge(), "Admin has 1 projects containing the search term " + newQuery); + assertEquals(1, searchResultPage.getTotalNumberOfSamplesByBadge(), "Admin has 1 samples containing the search term " + newQuery); } -// @Test -// public void testSampleSearch() { -// LoginPage.loginAsUser(driver()); -// page = SearchResultPage.initPage(driver()); -// -// page.globalSearch("samp", false); -// page = SearchResultPage.initPage(driver()); -// checkTranslations(page, ImmutableList.of("search"), null); -// -// page.waitForSearchResults(); -// int sampleCount = page.getSampleCount(); -// int projectCount = page.getProjectCount(); -// -// assertEquals(2, sampleCount, "should be 2 samples"); -// assertEquals(0, projectCount, "should be no projects"); -// -// } -// -// @Test -// public void testProjectSearch() { -// LoginPage.loginAsUser(driver()); -// page = SearchResultPage.initPage(driver()); -// -// page.globalSearch("project2", false); -// page = SearchResultPage.initPage(driver()); -// -// page.waitForSearchResults(); -// int sampleCount = page.getSampleCount(); -// int projectCount = page.getProjectCount(); -// -// assertEquals(0, sampleCount, "should be no samples"); -// assertEquals(1, projectCount, "should be 1 project"); -// -// page.globalSearch("ABCD", false); -// page = SearchResultPage.initPage(driver()); -// -// page.waitForSearchResults(); -// sampleCount = page.getSampleCount(); -// projectCount = page.getProjectCount(); -// -// assertEquals(0, sampleCount, "should be no samples"); -// assertEquals(0, projectCount, "should be no projects"); -// } + @Test + public void testGlobalSearchAsUser() { + LoginPage.loginAsUser(driver()); + SearchResultPage searchResultPage = new SearchResultPage(driver()); + final String query = "2"; + searchResultPage.enterSearchQueryInNavBar(query); + assertTrue(driver().getCurrentUrl().contains("search?query=" + query)); + assertEquals(query, searchResultPage.getSearchInputQuery()); + int numRows = searchResultPage.getTotalNumberOfProjectsInTable(); + assertEquals(2, numRows, "Expected number of projects"); + assertEquals(numRows, searchResultPage.getTotalNumberOfProjectsByBadge()); + assertEquals(1, searchResultPage.getTotalNumberOfSamplesByBadge(), "Expected total number of samples"); + assertFalse(searchResultPage.isAdminSearchTypeDisplayed(), "Users should not have the ability to switch between global and personal projects"); + + } } diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java index bc137f71584..ff12b07d8e2 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java @@ -1,13 +1,84 @@ package ca.corefacility.bioinformatics.irida.ria.integration.pages; +import org.openqa.selenium.By; +import org.openqa.selenium.Keys; import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; + +import java.time.Duration; /** * Result page for global search */ public class SearchResultPage extends AbstractPage { - public SearchResultPage(WebDriver driver) { - super(driver); - } + public SearchResultPage(WebDriver driver) { + super(driver); + } + + public String getSearchInputQuery() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + WebElement searchInput = wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//span[contains(@class, 't-search-input')]//input"))); + return searchInput.getAttribute("value"); + } + + public void enterNewSearchQuery(String query) { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + WebElement searchInput = wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//span[contains(@class, 't-search-input')]//input"))); + searchInput.sendKeys(Keys.chord(Keys.CONTROL, "a")); + searchInput.sendKeys(query); + wait.until(ExpectedConditions.urlContains(query)); + } + + public int getTotalNumberOfProjectsInTable() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + WebElement tbody = wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".t-search-projects tbody"))); + return tbody.findElements(By.tagName("tr")).size(); + } + + public int getTotalNumberOfProjectsByBadge() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + WebElement projectBadge; + try { + // This only works if there is an actual number greater than 0 + projectBadge = wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".t-search-nav [data-menu-id*=projects] .current"))); + + } catch (Exception e) { + projectBadge = wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".t-search-nav [data-menu-id*=projects] .t-count-badge sup"))); + } + return Integer.parseInt(projectBadge.getText()); + } + + public int getTotalNumberOfSamplesByBadge() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + WebElement projectBadge; + try { + // This only works if there is an actual number greater than 0 + projectBadge = wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".t-search-nav [data-menu-id*=samples] .current"))); + + } catch (Exception e) { + projectBadge = wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".t-search-nav [data-menu-id*=samples] .t-count-badge sup"))); + } + return Integer.parseInt(projectBadge.getText()); + } + + public boolean isAdminSearchTypeDisplayed() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + try { + wait.until(ExpectedConditions.visibilityOfElementLocated(By.className("t-admin-search-type"))); + return true; + } catch (Exception e) { + return false; + } + } + + public void selectAdminPersonalProject() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + WebElement dropdown = wait.until(ExpectedConditions.visibilityOfElementLocated(By.className("t-admin-search-type"))); + dropdown.click(); + WebElement personalButton = wait.until(ExpectedConditions.elementToBeClickable(By.cssSelector(".rc-virtual-list [title='PERSONAL']"))); + personalButton.click(); + } } diff --git a/src/test/resources/ca/corefacility/bioinformatics/irida/ria/web/ProjectsPageIT.xml b/src/test/resources/ca/corefacility/bioinformatics/irida/ria/web/ProjectsPageIT.xml index 676a15cea9c..fac61a3079c 100644 --- a/src/test/resources/ca/corefacility/bioinformatics/irida/ria/web/ProjectsPageIT.xml +++ b/src/test/resources/ca/corefacility/bioinformatics/irida/ria/web/ProjectsPageIT.xml @@ -26,6 +26,7 @@ + @@ -59,8 +60,10 @@ + + From 1ee66b0a26865b5c665bbce591ea43127d41e4d2 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 15 Nov 2022 10:50:31 -0600 Subject: [PATCH 393/655] starting to add unique key on label for metadata_field table --- .../irida/model/sample/MetadataTemplateField.java | 15 +++++++++------ .../changesets/unreleased/all-changes.xml | 4 +++- .../metadata-field-add-label-constraint.xml | 10 ++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/metadata-field-add-label-constraint.xml diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/model/sample/MetadataTemplateField.java b/src/main/java/ca/corefacility/bioinformatics/irida/model/sample/MetadataTemplateField.java index 1cbb08dc44b..4864b69bf5c 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/model/sample/MetadataTemplateField.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/model/sample/MetadataTemplateField.java @@ -1,19 +1,22 @@ package ca.corefacility.bioinformatics.irida.model.sample; -import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; -import org.hibernate.envers.Audited; -import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import java.util.List; +import java.util.Objects; import javax.persistence.*; import javax.validation.constraints.NotNull; -import java.util.List; -import java.util.Objects; + +import org.hibernate.envers.Audited; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; /** * Describes an individual field in a {@link MetadataTemplate}. */ @Entity -@Table(name = "metadata_field") +@Table(name = "metadata_field", + uniqueConstraints = @UniqueConstraint(columnNames = { "label" }, name = "UK_METADATA_FIELD_LABEL")) @Audited @EntityListeners(AuditingEntityListener.class) public class MetadataTemplateField { diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml index 61d30a2d3fa..ea58a9e4165 100644 --- a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/all-changes.xml @@ -1,4 +1,6 @@ \ No newline at end of file + http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd"> + + \ No newline at end of file diff --git a/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/metadata-field-add-label-constraint.xml b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/metadata-field-add-label-constraint.xml new file mode 100644 index 00000000000..0b2f2be52ba --- /dev/null +++ b/src/main/resources/ca/corefacility/bioinformatics/irida/database/changesets/unreleased/metadata-field-add-label-constraint.xml @@ -0,0 +1,10 @@ + + + + + + + \ No newline at end of file From c315867a308836b2933eb40e2ae809c8dd316833 Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Tue, 15 Nov 2022 12:14:47 -0600 Subject: [PATCH 394/655] quality of life updates --- .gitignore | 3 ++- CHANGELOG.md | 1 + doc/developer/setup/index.md | 50 ++++++++++++++++++++++++++++++++---- gradle.properties | 8 ++++++ 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 gradle.properties diff --git a/.gitignore b/.gitignore index cfa8596e02b..0abcc2e5398 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,5 @@ src/main/webapp/node_modules ### MISC ### /.theia/launch.json *.swp -/src/main/resources/application-local.properties \ No newline at end of file +/src/main/resources/application-local.properties +java_pid*.hprof diff --git a/CHANGELOG.md b/CHANGELOG.md index edf457a85a5..b5db55feb7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # Changelog ## [Unreleased] +* [Developer] Updated developer setup documentation, ignore java_pid*.hprof files, and added quality of life file `gradle.properties` ## [22.09.4] - 2022/11/14 * [REST]: Fixed issue with project/samples api response missing samples when a sample has a default sequencing object. See [PR 1413](https://github.com/phac-nml/irida/pull/1413) diff --git a/doc/developer/setup/index.md b/doc/developer/setup/index.md index 083bda3060c..0b768083432 100644 --- a/doc/developer/setup/index.md +++ b/doc/developer/setup/index.md @@ -78,15 +78,47 @@ IRIDA is configured to use the following credentials by default: You'll need to permit those user credentials to create the tables in the database. A quick one-liner for that is: - echo "grant all privileges on irida_test.* to 'test'@'localhost' identified by 'test';" | mysql -u root -p +``` +# Create the 'test' user: +echo "CREATE USER 'test'@'localhost' IDENTIFIED BY 'test';" | mysql -u root -p -Also create the database if it doesn't exist: +# Create the database if it doesn't exist: +echo "create database irida_test;" | mysql -u root -p - echo "create database irida_test;" | mysql -u root -p +# Grand 'test' user permissions to database: +echo "grant all privileges on irida_test.* to 'test'@'localhost';" | mysql -u root -p +``` -In order to run local integration tests, the database `irida_integration_test` is used instead. This database requires permissions from user `'test'@'localhost'`. To grant such permissions a quick one-liner is: +In order to run local integration tests, the same steps need to be performed for the database name `irida_integration_test` +``` +# Create the database if it doesn't exist: +echo "create database irida_integration_test;" | mysql -u root -p + +# Grand 'test' user permissions to database: +echo "grant all privileges on irida_integration_test.* to 'test'@'localhost';" | mysql -u root -p +``` + +### Configure Docker + +IRIDA uses Docker to create galaxy instances when running integration tests. + +https://docs.docker.com/engine/install/ubuntu/ + +You will also need to ensure your user has permissions to run docker commands. Please note that doing this effectively gives your user root access, so this should only be done in development environments. + +``` +# Create the docker group if it does not exist +sudo groupadd docker +# Add your user to the docker group. +sudo usermod -aG docker $USER +# login to docker group so you don't have to sign in/out +newgrp docker +# restart docker +sudo systemctl restart docker +# test that you can create docker instances +docker run hello-world +``` - echo "grant all privileges on irida_integration_test.* to 'test'@'localhost';" | mysql -u root -p ### Configure Filesystem Locations @@ -143,6 +175,14 @@ To run the full integration test suite for IRIDA please run the following: This will run all the integration test profiles using Gradle, and print out reports for each profile. +#### Memory requirements + +Your environment may throw java heap size errors when running tests. + +You can increase the heap size by editing the `gradle.properties` file in the project directory and adding the following line: + +`org.gradle.jvmargs=-Xmx4096m` + Setting up Galaxy ----------------- diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000000..521e750c2ab --- /dev/null +++ b/gradle.properties @@ -0,0 +1,8 @@ +# Project-wide Gradle settings. + +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +# org.gradle.jvmargs=-Xmx4096m From 7052de4d564c06a8f8ae9e780d6b1eea1350d510 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Tue, 15 Nov 2022 14:26:49 -0600 Subject: [PATCH 395/655] replacing select with radio group buttons in TargetMetadataRestrictions.tsx --- .../metadata/TargetMetadataRestriction.tsx | 19 ++++++++++++------- .../CreateProjectMetadataRestrictions.jsx | 1 + .../js/pages/projects/share/ShareMetadata.jsx | 6 +++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/webapp/resources/js/components/metadata/TargetMetadataRestriction.tsx b/src/main/webapp/resources/js/components/metadata/TargetMetadataRestriction.tsx index 73fe2bba771..1da0015b329 100644 --- a/src/main/webapp/resources/js/components/metadata/TargetMetadataRestriction.tsx +++ b/src/main/webapp/resources/js/components/metadata/TargetMetadataRestriction.tsx @@ -1,4 +1,4 @@ -import { Form, Popover, Select, Space, Tag, Tooltip, Typography } from "antd"; +import { Form, Popover, Radio, Space, Tag, Tooltip, Typography } from "antd"; import React from "react"; import { useDispatch } from "react-redux"; import { IconInfoCircle, IconWarningOutlined } from "../icons/Icons"; @@ -117,17 +117,22 @@ export function TargetMetadataRestriction({ visible={tooltipVisible} > - + ); diff --git a/src/main/webapp/resources/js/pages/projects/create/CreateProjectMetadataRestrictions.jsx b/src/main/webapp/resources/js/pages/projects/create/CreateProjectMetadataRestrictions.jsx index eb4e73152a3..4f33a8694bc 100644 --- a/src/main/webapp/resources/js/pages/projects/create/CreateProjectMetadataRestrictions.jsx +++ b/src/main/webapp/resources/js/pages/projects/create/CreateProjectMetadataRestrictions.jsx @@ -110,6 +110,7 @@ export function CreateProjectMetadataRestrictions({ form }) { dataSource={metadataRestrictions} scroll={{ y: 300 }} pagination={false} + tableLayout="auto" /> ) : ( ) : ( Date: Wed, 16 Nov 2022 10:23:45 -0600 Subject: [PATCH 396/655] fixing test --- .../irida/ria/integration/pages/CreateProjectComponent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java index 81dc5246f6e..059edc0d7e7 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java @@ -151,7 +151,7 @@ public boolean correctMetadataFieldDataDisplayed() { currentRestrictions.add(driver.findElement(By.className("t-current-restriction-" + label)).getText()); } - List targetRestrictionsText = driver.findElements(By.className("ant-select-selection-item")); + List targetRestrictionsText = driver.findElements(By.className("ant-radio-button-wrapper-checked")); for (WebElement element : targetRestrictionsText) { if (!element.getText().isEmpty()) From 9dafa494bbb2520c0ba8c06c4a5bb4b1dd576f3d Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 16 Nov 2022 10:51:28 -0600 Subject: [PATCH 397/655] renaming 'Upload File' to 'Select File' --- src/main/resources/i18n/messages.properties | 12 ++++++------ ...adFile.tsx => SampleMetadataImportSelectFile.tsx} | 10 +++++----- .../pages/projects/samples-metadata-import/index.js | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) rename src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/{SampleMetadataImportUploadFile.tsx => SampleMetadataImportSelectFile.tsx} (91%) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index f8aa6a582c9..ea2ea299e11 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -1984,19 +1984,19 @@ server.metadataimport.results.save.success.multiple-created=\ and {0} new sample SampleMetadataImportWizard.title=Sample Metadata Uploader SampleMetadataImportWizard.intro=A tool you can use to upload excel or csv files containing metadata for samples in this project. -SampleMetadataImportSteps.step1=Upload File +SampleMetadataImportSteps.step1=Select File SampleMetadataImportSteps.step2=Map Columns SampleMetadataImportSteps.step3=Review Data SampleMetadataImportSteps.step4=Complete -SampleMetadataImportUploadFile.warning=Metadata uploaded will overwrite duplicate metadata on the sample. -SampleMetadataImportUploadFile.dropzone=Click or drop Excel/CSV file containing metadata for samples in this project. -SampleMetadataImportUploadFile.success={0} was uploaded successfully. -SampleMetadataImportUploadFile.error={0} failed to upload. +SampleMetadataImportSelectFile.warning=Metadata uploaded will overwrite duplicate metadata on the sample. +SampleMetadataImportSelectFile.dropzone=Click or drop Excel/CSV file containing metadata for samples in this project. +SampleMetadataImportSelectFile.success={0} was uploaded successfully. +SampleMetadataImportSelectFile.error={0} failed to upload. SampleMetadataImportMapColumns.form.sampleNameColumn=Select Sample Name Column SampleMetadataImportMapColumns.form.metadataColumns=Review Parsed Metadata Columns -SampleMetadataImportMapColumns.button.back=Upload a new file +SampleMetadataImportMapColumns.button.back=Select a new file SampleMetadataImportMapColumns.button.next=Review the data SampleMetadataImportMapColumns.table.empty=Waiting on the selection of the sample name column. SampleMetadataImportMapColumns.table.header=Header diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx similarity index 91% rename from src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx rename to src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx index 87c48fe45a9..a208766816a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportUploadFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx @@ -19,7 +19,7 @@ const { Text } = Typography; * @returns {*} * @constructor */ -export function SampleMetadataImportUploadFile(): JSX.Element { +export function SampleMetadataImportSelectFile(): JSX.Element { const { projectId } = useParams<{ projectId: string }>(); const navigate: NavigateFunction = useNavigate(); const dispatch: ImportDispatch = useImportDispatch(); @@ -75,7 +75,7 @@ export function SampleMetadataImportUploadFile(): JSX.Element { if (status === "done") { notification.success({ message: i18n( - "SampleMetadataImportUploadFile.success", + "SampleMetadataImportSelectFile.success", info.file.name ), }); @@ -84,7 +84,7 @@ export function SampleMetadataImportUploadFile(): JSX.Element { setLoading(false); setStatus("error"); notification.error({ - message: i18n("SampleMetadataImportUploadFile.error", info.file.name), + message: i18n("SampleMetadataImportSelectFile.error", info.file.name), }); } }, @@ -94,9 +94,9 @@ export function SampleMetadataImportUploadFile(): JSX.Element { {i18n("SampleMetadataImportUploadFile.warning")} + {i18n("SampleMetadataImportSelectFile.warning")} } options={options} props={{ className: "t-metadata-uploader-dropzone" }} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js index d6649bfeb51..9f8925127fa 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/index.js @@ -5,7 +5,7 @@ import { render } from "react-dom"; import { SampleMetadataImportComplete } from "./components/SampleMetadataImportComplete"; import { SampleMetadataImportMapColumns } from "./components/SampleMetadataImportMapColumns"; import { SampleMetadataImportReview } from "./components/SampleMetadataImportReview"; -import { SampleMetadataImportUploadFile } from "./components/SampleMetadataImportUploadFile"; +import { SampleMetadataImportSelectFile } from "./components/SampleMetadataImportSelectFile"; import { setBaseUrl } from "../../../utilities/url-utilities"; import store from "./redux/store"; @@ -20,7 +20,7 @@ render( } + element={} /> Date: Wed, 16 Nov 2022 14:07:33 -0600 Subject: [PATCH 398/655] changing ant design upload options so it does not do a post request --- src/main/resources/i18n/messages.properties | 2 +- .../SampleMetadataImportSelectFile.tsx | 35 ++++++------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index ea2ea299e11..43e5b30df6f 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -1991,7 +1991,7 @@ SampleMetadataImportSteps.step4=Complete SampleMetadataImportSelectFile.warning=Metadata uploaded will overwrite duplicate metadata on the sample. SampleMetadataImportSelectFile.dropzone=Click or drop Excel/CSV file containing metadata for samples in this project. -SampleMetadataImportSelectFile.success={0} was uploaded successfully. +SampleMetadataImportSelectFile.success={0} was read successfully. SampleMetadataImportSelectFile.error={0} failed to upload. SampleMetadataImportMapColumns.form.sampleNameColumn=Select Sample Name Column diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx index a208766816a..33b98eb2f7a 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx @@ -9,7 +9,7 @@ import { WorkBook } from "xlsx"; import { ImportDispatch, useImportDispatch } from "../redux/store"; import { NavigateFunction } from "react-router/dist/lib/hooks"; import { MetadataItem } from "../../../../apis/projects/samples"; -import { RcFile, UploadFileStatus } from "antd/lib/upload/interface"; +import { RcFile } from "antd/lib/upload/interface"; const { Text } = Typography; @@ -36,15 +36,8 @@ export function SampleMetadataImportSelectFile(): JSX.Element { multiple: false, showUploadList: false, accept: ".xls,.xlsx,.csv", - onChange(info: { - file: { - originFileObj?: RcFile; - name?: string; - status?: UploadFileStatus; - }; - }) { - const { status } = info.file; - if (status !== "uploading") { + beforeUpload: (file: RcFile) => { + if (file) { setLoading(true); const reader = new FileReader(); if (reader.readAsBinaryString) { @@ -67,26 +60,20 @@ export function SampleMetadataImportSelectFile(): JSX.Element { await dispatch(setHeaders({ headers: Object.keys(cleanRows[0]) })); await dispatch(setMetadata(cleanRows)); }; - if (info.file.originFileObj) { - reader.readAsBinaryString(info.file.originFileObj); - } + reader.readAsBinaryString(file); + notification.success({ + message: i18n("SampleMetadataImportSelectFile.success", file.name), + }); + navigate(`/${projectId}/sample-metadata/upload/columns`); } - } - if (status === "done") { - notification.success({ - message: i18n( - "SampleMetadataImportSelectFile.success", - info.file.name - ), - }); - navigate(`/${projectId}/sample-metadata/upload/columns`); - } else if (status === "error") { + } else { setLoading(false); setStatus("error"); notification.error({ - message: i18n("SampleMetadataImportSelectFile.error", info.file.name), + message: i18n("SampleMetadataImportSelectFile.error", file.name), }); } + return false; }, }; From 5edd11e643194e181ecd458ebc7c4ec450a24694 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 16 Nov 2022 14:13:54 -0600 Subject: [PATCH 399/655] changing ant design upload options so it does not do a post request --- .../components/SampleMetadataImportSelectFile.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx index 33b98eb2f7a..22e5e423a3b 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportSelectFile.tsx @@ -37,7 +37,7 @@ export function SampleMetadataImportSelectFile(): JSX.Element { showUploadList: false, accept: ".xls,.xlsx,.csv", beforeUpload: (file: RcFile) => { - if (file) { + try { setLoading(true); const reader = new FileReader(); if (reader.readAsBinaryString) { @@ -66,7 +66,7 @@ export function SampleMetadataImportSelectFile(): JSX.Element { }); navigate(`/${projectId}/sample-metadata/upload/columns`); } - } else { + } catch (error) { setLoading(false); setStatus("error"); notification.error({ From ef78aebf7ea49844c0bfc32127a5c03f414ac32c Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 16 Nov 2022 14:29:35 -0600 Subject: [PATCH 400/655] making create project modal wider --- .../resources/js/pages/projects/create/CreateProjectLayout.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/projects/create/CreateProjectLayout.jsx b/src/main/webapp/resources/js/pages/projects/create/CreateProjectLayout.jsx index 32d7500000f..7e14de6ed13 100644 --- a/src/main/webapp/resources/js/pages/projects/create/CreateProjectLayout.jsx +++ b/src/main/webapp/resources/js/pages/projects/create/CreateProjectLayout.jsx @@ -138,7 +138,7 @@ export function CreateProjectLayout({ children }) {
    } onCancel={onCancel} - width={720} + width={900} title={i18n("CreateProject.title")} > From 7fb6846a95082ca71713f1d7503e55c5bf83681c Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Wed, 16 Nov 2022 14:58:25 -0600 Subject: [PATCH 401/655] updating project settings metadata fields to use radio group buttons --- .../metadata/MetadataFieldsListManager.jsx | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/settings/components/metadata/MetadataFieldsListManager.jsx b/src/main/webapp/resources/js/pages/projects/settings/components/metadata/MetadataFieldsListManager.jsx index 5d1b41163bd..d0086c0c374 100644 --- a/src/main/webapp/resources/js/pages/projects/settings/components/metadata/MetadataFieldsListManager.jsx +++ b/src/main/webapp/resources/js/pages/projects/settings/components/metadata/MetadataFieldsListManager.jsx @@ -1,4 +1,4 @@ -import { Button, Empty, notification, Select, Space, Table } from "antd"; +import { Button, Empty, notification, Radio, Space, Table } from "antd"; import React from "react"; import { getMetadataRestrictions, @@ -24,9 +24,8 @@ export default function MetadataFieldsListManager() { isLoading, refetch: refetchFields, } = useGetMetadataFieldsForProjectQuery(projectId); - const [ - updateProjectMetadataFieldRestriction, - ] = useUpdateProjectMetadataFieldRestrictionMutation(); + const [updateProjectMetadataFieldRestriction] = + useUpdateProjectMetadataFieldRestrictionMutation(); const [{ selected, selectedItems }, { setSelected }] = useTableSelect(fields); @@ -75,13 +74,25 @@ export default function MetadataFieldsListManager() { key: "restriction", render(restriction, field) { return ( - - {metadataRoles?.map((role: MetadataRoles) => ( - + {/* Styles can be replaced with compact space in the future */} + {metadataRoles.map((role: MetadataRoles) => ( + {role.label} - + ))} - + From 20f092fee7704c5960b84eedbad3233ef7f1b32a Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 17 Nov 2022 14:55:22 -0600 Subject: [PATCH 408/655] fixing tests --- .../pages/CreateProjectComponent.java | 2 +- .../pages/projects/ProjectMetadataPage.java | 32 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java index 059edc0d7e7..346ce1a289d 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/CreateProjectComponent.java @@ -168,7 +168,7 @@ public boolean correctMetadataFieldDataDisplayed() { public boolean correctMetadataFieldDataDisplayedForNewProject() { int mainIndex = 0; int textIndex = 0; - List restrictions = driver.findElements(By.className("ant-select-selection-item")); + List restrictions = driver.findElements(By.className("ant-radio-button-wrapper-checked")); for (WebElement element : projectMetadataFieldRows) { String labelText = element.findElement(By.className("t-m-field-label")).getText(); diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectMetadataPage.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectMetadataPage.java index 6715c42f093..ddf8c992255 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectMetadataPage.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/projects/ProjectMetadataPage.java @@ -40,7 +40,7 @@ public class ProjectMetadataPage extends AbstractPage { WebElement templateEditName; @FindBy(className = "t-field-restriction") - List fieldRestrictionSelects; + List fieldRestrictions; public ProjectMetadataPage(WebDriver driver) { super(driver); @@ -118,8 +118,8 @@ public void deleteTemplate(String name) { } } templateRow.findElement(By.className("t-t-remove-button")).click(); - WebElement confirm = wait - .until(ExpectedConditions.visibilityOfElementLocated(By.className("t-t-confirm-remove"))); + WebElement confirm = wait.until( + ExpectedConditions.visibilityOfElementLocated(By.className("t-t-confirm-remove"))); confirm.click(); wait.until(ExpectedConditions.stalenessOf(confirm)); } @@ -183,35 +183,33 @@ public boolean removeButtonIsDisabled() { public boolean defaultTemplateTagVisible() { WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(4)); - WebElement defaultTag = wait - .until(ExpectedConditions.visibilityOfElementLocated(By.className("t-t-default-tag"))); + WebElement defaultTag = wait.until( + ExpectedConditions.visibilityOfElementLocated(By.className("t-t-default-tag"))); return defaultTag.isDisplayed(); } public boolean setDefaultTemplateButtonVisible() { WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(4)); - WebElement setDefaultTemplateBtn = wait - .until(ExpectedConditions.visibilityOfElementLocated(By.className("t-t-set-default-button"))); + WebElement setDefaultTemplateBtn = wait.until( + ExpectedConditions.visibilityOfElementLocated(By.className("t-t-set-default-button"))); return setDefaultTemplateBtn.isDisplayed(); } public boolean areFieldRestrictionSettingsVisible() { - return fieldRestrictionSelects.size() > 0; + return fieldRestrictions.size() > 0; } public String getFieldRestrictionForRow(int row) { - return fieldRestrictionSelects.get(row).findElement(By.className("ant-select-selection-item")).getText(); + return fieldRestrictions.get(row).findElement(By.className("ant-radio-button-wrapper-checked")).getText(); } public void updateFieldRestrictionToLevel(int row, int optionNumber) { - WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(2)); - fieldRestrictionSelects.get(row).click(); - WebElement dropdown = wait - .until(ExpectedConditions.visibilityOfElementLocated(By.className("ant-select-dropdown"))); - List options = dropdown.findElements(By.className("ant-select-item-option")); - options.get(optionNumber).click(); - wait.until(ExpectedConditions.invisibilityOf(dropdown)); - + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(4)); + WebElement radioButton = fieldRestrictions.get(row) + .findElements(By.className("ant-radio-button-wrapper")) + .get(optionNumber); + radioButton.click(); + wait.until(ExpectedConditions.attributeContains(radioButton, "class", "ant-radio-button-wrapper-checked")); } private void waitForFields() { From e5b78def3275cb3a2798166712258d44e4c33eb9 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 18 Nov 2022 08:12:01 -0600 Subject: [PATCH 409/655] chore: Updated i18n --- src/main/resources/i18n/messages.properties | 26 +++++++++---------- .../js/pages/search/SearchLayout.tsx | 16 +++++++----- .../js/pages/search/SearchProjectsTable.tsx | 10 +++---- .../js/pages/search/SearchSamplesTable.tsx | 10 +++---- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index 0baff878f0f..007c3b976c6 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2178,20 +2178,18 @@ page.cart.title=Cart # Search # # ========================================================================================== # search.title=Search -search.samples.tab=Samples -search.projects.tab=Projects -search.samples.name=Sample Name -search.samples.organism=Organism -search.samples.project=Project -search.samples.created=Created -search.samples.modified=Modified -search.projects.id=ID -search.projects.name=Project Name -search.projects.organism=Organism -search.projects.samples=Samples -search.projects.created=Created -search.projects.modified=Modified -search.admin.hint=You are currently searching your personal projects/samples. Click here to search all projects/samples. +SearchLayout.tooltip=What are you searching for? +SearchLayout.projects=Projects +SearchLayout.samples=Samples +SearchLayout.admin.global=Everywhere +SearchLayout.admin.personal=Personal +SearchTable.name=Name +SearchTable.organism=Organism +SearchTable.created=Create +SearchTable.updated=Updated +SearchProjectsTable.samples=# Samples +SearchSamplesTable.project=Projects + # ========================================================================================== # # SAMPLE DETAILS COMPONENT # # ========================================================================================== # diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index 6ceac04d728..f63fc9dc3f4 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -195,7 +195,7 @@ export default function SearchLayout() { alignItems: "center", }} > - PROJECTS + {i18n("SearchLayout.projects").toUpperCase()} ), @@ -211,7 +211,7 @@ export default function SearchLayout() { alignItems: "center", }} > - SAMPLES + {i18n("SearchLayout.samples").toUpperCase()} ), @@ -228,14 +228,18 @@ export default function SearchLayout() { onChange={(value) => setGlobal(value === "global")} className="t-admin-search-type" > - PERSONAL - GLOBAL + + {i18n("SearchLayout.admin.global")} + + + {i18n("SearchLayout.admin.personal")} + ); return ( - What are you searching for? + {i18n("SearchLayout.tooltip")} , sorter: true, }, { key: `organism`, dataIndex: `organism`, - title: "ORGANISM", + title: i18n("SearchTable.organism"), sorter: true, }, { key: `samples`, dataIndex: `samples`, - title: `SAMPLES`, + title: i18n("SearchProjectsTable.samples"), width: 150, }, { key: `createdDate`, dataIndex: `createdDate`, - title: `CREATED DATE`, + title: i18n("SearchTable.created"), render: (text: string) => formatInternationalizedDateTime(text), sorter: true, width: 200, @@ -58,7 +58,7 @@ export default function SearchProjectsTable({ { key: `modifiedDate`, dataIndex: `modifiedDate`, - title: `LAST UPDATED`, + title: i18n("SearchTable.updated"), render: (text: string) => formatInternationalizedDateTime(text), sorter: true, defaultSortOrder: "descend", diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx index 8e65e160b15..bbecf084dde 100644 --- a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -32,7 +32,7 @@ export default function SearchSamplesTable({ { key: `sampleName`, dataIndex: "name", - title: "NAME", + title: i18n("SearchTable.name"), sorter: true, render: (name, sample) => { return ( @@ -48,13 +48,13 @@ export default function SearchSamplesTable({ { key: `organism`, dataIndex: `organism`, - title: "ORGANISM", + title: i18n("SearchTable.organism"), sorter: true, }, { key: `projects`, dataIndex: `projects`, - title: `PROJECTS`, + title: i18n("SearchSamplesTable.project"), render: (projects: SearchProject[]) => { return projects.map((project) => ( @@ -64,7 +64,7 @@ export default function SearchSamplesTable({ { key: `createdDate`, dataIndex: `createdDate`, - title: `CREATED DATE`, + title: i18n("SearchTable.created"), render: (text: string) => formatInternationalizedDateTime(text), sorter: true, width: 200, @@ -72,7 +72,7 @@ export default function SearchSamplesTable({ { key: `modifiedDate`, dataIndex: `modifiedDate`, - title: `MODIFIED DATE`, + title: i18n("SearchTable.updated"), render: (text: string) => formatInternationalizedDateTime(text), sorter: true, defaultSortOrder: "descend", From 74ccbaca5697716973775535516c64cb02251053 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 18 Nov 2022 08:12:34 -0600 Subject: [PATCH 410/655] chore: Removed bootstrap-datepicker dependency --- src/main/webapp/package.json | 1 - src/main/webapp/pnpm-lock.yaml | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/main/webapp/package.json b/src/main/webapp/package.json index 4c0f8e421b1..0629429dadb 100644 --- a/src/main/webapp/package.json +++ b/src/main/webapp/package.json @@ -40,7 +40,6 @@ "antd": "4.19.5", "axios": "^0.26.1", "bootstrap": "3.4.1", - "bootstrap-daterangepicker": "^3.1.0", "clipboard": "^1.7.1", "dayjs": "^1.11.1", "deck.gl": "^8.7.12", diff --git a/src/main/webapp/pnpm-lock.yaml b/src/main/webapp/pnpm-lock.yaml index 4fe9bf3f095..939276c17f7 100644 --- a/src/main/webapp/pnpm-lock.yaml +++ b/src/main/webapp/pnpm-lock.yaml @@ -53,7 +53,6 @@ specifiers: babel-loader: ^8.2.4 babel-plugin-import: ^1.13.5 bootstrap: 3.4.1 - bootstrap-daterangepicker: ^3.1.0 browserslist: ^4.20.2 clipboard: ^1.7.1 css-loader: ^6.7.1 @@ -142,7 +141,6 @@ dependencies: antd: 4.19.5_sfoxds7t5ydpegc3knd667wn6m axios: 0.26.1 bootstrap: 3.4.1 - bootstrap-daterangepicker: 3.1.0 clipboard: 1.7.1 dayjs: 1.11.1 deck.gl: 8.7.12_qdym3cmqy3izoldsblxf2ftyey @@ -3830,13 +3828,6 @@ packages: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} dev: true - /bootstrap-daterangepicker/3.1.0: - resolution: {integrity: sha512-oaQZx6ZBDo/dZNyXGVi2rx5GmFXThyQLAxdtIqjtLlYVaQUfQALl5JZMJJZzyDIX7blfy4ppZPAJ10g8Ma4d/g==} - dependencies: - jquery: 3.5.1 - moment: 2.29.2 - dev: false - /bootstrap/3.4.1: resolution: {integrity: sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==} engines: {node: '>=6'} From 7814ceaedf65b339a6d0c221619ec0fc3a5e8520 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 18 Nov 2022 08:12:56 -0600 Subject: [PATCH 411/655] test: Update number os users on project since I added a user for another test --- .../integration/projects/ProjectMembersPageIT.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java index 4c06f66ad55..e06298eab1c 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java @@ -1,7 +1,5 @@ package ca.corefacility.bioinformatics.irida.ria.integration.projects; -import org.junit.jupiter.api.Test; - import ca.corefacility.bioinformatics.irida.model.enums.ProjectMetadataRole; import ca.corefacility.bioinformatics.irida.model.enums.ProjectRole; import ca.corefacility.bioinformatics.irida.ria.integration.AbstractIridaUIITChromeDriver; @@ -11,8 +9,8 @@ import ca.corefacility.bioinformatics.irida.ria.integration.pages.projects.ProjectDetailsPage; import ca.corefacility.bioinformatics.irida.ria.integration.pages.projects.ProjectSyncPage; import ca.corefacility.bioinformatics.irida.ria.integration.utilities.RemoteApiUtilities; - import com.github.springtestdbunit.annotation.DatabaseSetup; +import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -29,22 +27,22 @@ public void testCanManagePageSetUp() { LoginPage.loginAsManager(driver()); ProjectMembersPage page = ProjectMembersPage.goTo(driver()); assertEquals("Members", page.getPageHeaderTitle(), "Check for proper translation in title"); - assertEquals(2, page.getNumberOfMembers(), "Should be 2 members in the project"); + assertEquals(3, page.getNumberOfMembers(), "Should be 3 members in the project"); assertTrue(page.isAddMemberBtnVisible(), "Add Members button should be visible"); // Test remove user from project page.removeUser(1); assertTrue(page.isNotificationDisplayed()); - assertEquals(1, page.getNumberOfMembers(), "Should be 1 member in the project"); + assertEquals(2, page.getNumberOfMembers(), "Should be 2 members in the project"); // Should not be able to remove the manager so the remove button should be disabled assertFalse(page.lastManagerRemoveButtonEnabled(0)); - assertEquals(1, page.getNumberOfMembers(), "Should be 1 member in the project"); + assertEquals(2, page.getNumberOfMembers(), "Should be 2 member in the project"); // Test Add user to project page.addUserToProject("test"); page.isNotificationDisplayed(); - assertEquals(2, page.getNumberOfMembers(), "Should be 2 members in the project"); + assertEquals(3, page.getNumberOfMembers(), "Should be 3 members in the project"); // Try updating the users role to owner page.updateUserRole(0, ProjectRole.PROJECT_OWNER.toString()); From 56328471914e64303e5f1f90c487500e63337e86 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 18 Nov 2022 09:26:52 -0600 Subject: [PATCH 412/655] fix: Fixed array declaration --- .../irida/ria/integration/projects/ProjectMembersPageIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java index e06298eab1c..25444e22c2a 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/projects/ProjectMembersPageIT.java @@ -94,7 +94,7 @@ public void testRemoteProjectManagerPageSetup() { assertFalse(url.isEmpty(), "URL should not be empty"); page.submitProject(); - String pathTokens[] = driver().getCurrentUrl().split("/"); + String[] pathTokens = driver().getCurrentUrl().split("/"); Long projectId = Long.valueOf(pathTokens[pathTokens.length - 1]); ProjectMembersPage remoteProjectMembersPage = ProjectMembersPage.goToRemoteProject(driver(), projectId); From 64232b45987bc6f9c74fb4f60408372ee6cbebbb Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Fri, 18 Nov 2022 10:18:43 -0600 Subject: [PATCH 413/655] fix: Fixed test due to changed internationalization --- .../irida/ria/integration/pages/SearchResultPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java index d3f08842ba3..78e0745fc3f 100644 --- a/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java +++ b/src/test/java/ca/corefacility/bioinformatics/irida/ria/integration/pages/SearchResultPage.java @@ -78,7 +78,7 @@ public void selectAdminPersonalProject() { WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); WebElement dropdown = wait.until(ExpectedConditions.visibilityOfElementLocated(By.className("t-admin-search-type"))); dropdown.click(); - WebElement personalButton = wait.until(ExpectedConditions.elementToBeClickable(By.cssSelector(".rc-virtual-list [title='PERSONAL']"))); + WebElement personalButton = wait.until(ExpectedConditions.elementToBeClickable(By.cssSelector(".rc-virtual-list [title='Personal']"))); personalButton.click(); } } From 8b237f90d0c4c5508d0e7cfe7fad3ce4f5e2269c Mon Sep 17 00:00:00 2001 From: Aaron Petkau Date: Fri, 18 Nov 2022 11:10:02 -0600 Subject: [PATCH 414/655] Fixed broken links in documentation --- CHANGELOG.md | 3 ++- doc/developer/getting-started/index.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e4d97c5786..50a47d67d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ ## [Unreleased] * [UI/Developer]: Updated `react-router` to the version 6.4.3. See[PR 1405](https://github.com/phac-nml/irida/pull/1405) -* [Developer] Updated developer setup documentation, ignore java_pid*.hprof files, and added quality of life file `gradle.properties` +* [Developer] Updated developer setup documentation, ignore java_pid\*.hprof files, and added quality of life file `gradle.properties`. See [PR 1415](https://github.com/phac-nml/irida/pull/1415). +* [Documentation]: Updated broken links in developer documentation. See [PR 1416](https://github.com/phac-nml/irida/pull/1416). ## [22.09.4] - 2022/11/14 * [REST]: Fixed issue with project/samples api response missing samples when a sample has a default sequencing object. See [PR 1413](https://github.com/phac-nml/irida/pull/1413) diff --git a/doc/developer/getting-started/index.md b/doc/developer/getting-started/index.md index 6317a385c9a..e15208ace84 100644 --- a/doc/developer/getting-started/index.md +++ b/doc/developer/getting-started/index.md @@ -66,7 +66,7 @@ Liquibase is used to manage IRIDA's relational database change management. Any Documentation: [https://docs.galaxyproject.org/en/master/index.html](https://docs.galaxyproject.org/en/master/index.html) -Galaxy is used as IRIDA's analysis workflow engine. Analysis pipelines must be developed as Galaxy pipelines to integrate with IRIDA's workflow system. See the [Galaxy Setup](/administrator/galaxy/) documentation for Galaxy installation and the [Tool Development](/developer/tools/) documentation for building tools for IRDIA. +Galaxy is used as IRIDA's analysis workflow engine. Analysis pipelines must be developed as Galaxy pipelines to integrate with IRIDA's workflow system. See the [Galaxy Setup](../../administrator/galaxy/) documentation for Galaxy installation and the [Tool Development](../../developer/tools/) documentation for building tools for IRDIA. #### Other important libraries {:.no_toc} @@ -266,7 +266,7 @@ IRIDA documentation can be found in the ##### Testing IRIDA documentation locally To view the documentation locally or make changes, you can checkout the above GitHub project and make changes. To run the server locally you can run Jekyll to generate the pages. -First `cd` into the `docs/` directory and run the following command: +First `cd` into the `doc/` directory and run the following command: ```bash bundle exec jekyll serve From 99c95aeff9a5f6f5b6884b2cd567d1c15c932e79 Mon Sep 17 00:00:00 2001 From: Aaron Petkau Date: Fri, 18 Nov 2022 11:11:57 -0600 Subject: [PATCH 415/655] Fixed up PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a47d67d5a..59c707a1586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## [Unreleased] * [UI/Developer]: Updated `react-router` to the version 6.4.3. See[PR 1405](https://github.com/phac-nml/irida/pull/1405) * [Developer] Updated developer setup documentation, ignore java_pid\*.hprof files, and added quality of life file `gradle.properties`. See [PR 1415](https://github.com/phac-nml/irida/pull/1415). -* [Documentation]: Updated broken links in developer documentation. See [PR 1416](https://github.com/phac-nml/irida/pull/1416). +* [Documentation]: Updated broken links in developer documentation. See [PR 1418](https://github.com/phac-nml/irida/pull/1418). ## [22.09.4] - 2022/11/14 * [REST]: Fixed issue with project/samples api response missing samples when a sample has a default sequencing object. See [PR 1413](https://github.com/phac-nml/irida/pull/1413) From 15db85278098c4705adf6d66253aabd9e034f2ea Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 18 Nov 2022 15:19:47 -0600 Subject: [PATCH 416/655] getting the locked samples for a project --- .../web/ajax/dto/LockedSamplesResponse.java | 21 ++++++++++++++++++ .../ProjectSamplesAjaxController.java | 17 ++++++++++---- .../web/services/UIProjectSampleService.java | 17 ++++++++++++-- .../resources/js/apis/projects/samples.ts | 13 +++++++++++ .../redux/importReducer.ts | 22 ++++++++++++++----- 5 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/LockedSamplesResponse.java diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/LockedSamplesResponse.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/LockedSamplesResponse.java new file mode 100644 index 00000000000..42d9615afc0 --- /dev/null +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/dto/LockedSamplesResponse.java @@ -0,0 +1,21 @@ +package ca.corefacility.bioinformatics.irida.ria.web.ajax.dto; + +import java.util.List; + +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxResponse; + +/** + * Returns an ajax response with locked sample ids. + */ +public class LockedSamplesResponse extends AjaxResponse { + private List sampleIds; + + public LockedSamplesResponse(List sampleIds) { + this.sampleIds = sampleIds; + } + + public List getSampleIds() { + return sampleIds; + } +} + diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java index ff333bc5436..1004762ab75 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/ajax/projects/ProjectSamplesAjaxController.java @@ -12,10 +12,8 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.CreateSampleRequest; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.SampleFilesResponse; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.SampleNameValidationResponse; -import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.UpdateSampleRequest; +import ca.corefacility.bioinformatics.irida.model.sample.Sample; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.*; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxErrorResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxSuccessResponse; @@ -226,4 +224,15 @@ public ResponseEntity validateSampleNames(@PathVariable Long proje return ResponseEntity.ok(uiProjectSampleService.validateSampleNames(projectId, request)); } + /** + * Get a list of {@link Sample} ids that are locked in the given project + * + * @param projectId project identifier + * @return a boolean + */ + @GetMapping("/locked") + public ResponseEntity getLockedSamplesInProject(@PathVariable Long projectId) { + return ResponseEntity.ok(uiProjectSampleService.getLockedSamplesInProject(projectId)); + } + } diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java index a892ebf67b3..cfe6e66c4d0 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UIProjectSampleService.java @@ -17,6 +17,7 @@ import ca.corefacility.bioinformatics.irida.model.sample.Sample; import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.CreateSampleRequest; +import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.LockedSamplesResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.SampleNameValidationResponse; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.UpdateSampleRequest; import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.ajax.AjaxCreateItemSuccessResponse; @@ -109,7 +110,6 @@ public ResponseEntity validateNewSampleName(String // Check to see if the sample name already exists. try { - Project project = projectService.read(projectId); sampleService.getSampleBySampleName(project, name); return ResponseEntity.status(HttpStatus.CONFLICT) @@ -132,8 +132,8 @@ public ResponseEntity validateNewSampleName(String */ @Transactional public ResponseEntity createSample(CreateSampleRequest request, Long projectId, Locale locale) { - Project project = projectService.read(projectId); try { + Project project = projectService.read(projectId); Sample sample = new Sample(request.getName()); if (!Strings.isNullOrEmpty(request.getOrganism())) { sample.setOrganism(request.getOrganism()); @@ -184,6 +184,18 @@ public ResponseEntity updateSample(UpdateSampleRequest request, Lo } } + /** + * Get a list of {@link Sample} ids that are locked in the given project + * + * @param projectId project identifier + * @return result of creating the sample + */ + public LockedSamplesResponse getLockedSamplesInProject(Long projectId) { + Project project = projectService.read(projectId); + List lockedSampleIds = sampleService.getLockedSamplesInProject(project); + return new LockedSamplesResponse(lockedSampleIds); + } + /** * Creates a metadata entry set for a sample, assuming the metadata field and restriction exist * @@ -198,4 +210,5 @@ private Set createMetadata(List metadataField }).collect(Collectors.toSet()); return metadataEntrySet; } + } diff --git a/src/main/webapp/resources/js/apis/projects/samples.ts b/src/main/webapp/resources/js/apis/projects/samples.ts index 6a6ad1c7df2..a9a44ac5992 100644 --- a/src/main/webapp/resources/js/apis/projects/samples.ts +++ b/src/main/webapp/resources/js/apis/projects/samples.ts @@ -21,6 +21,10 @@ export interface ValidateSamplesResponse { samples: ValidateSampleNameModel[]; } +export interface LockedSamplesResponse { + sampleIds: number[]; +} + export interface MetadataItem { [field: string]: string; rowKey: string; @@ -116,6 +120,15 @@ export async function validateSamples({ return response.data; } +export async function getLockedSamples({ + projectId, +}: { + projectId: string; +}): Promise { + const response = await axios.get(`${URL}/${projectId}/samples/locked`); + return response.data; +} + export async function createSample({ projectId, body, diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts index 5dc28c0ef92..b9c9d84b505 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts @@ -7,6 +7,8 @@ import { validateSampleName } from "../../../../apis/metadata/sample-utils"; import { createSample, FieldUpdate, + getLockedSamples, + LockedSamplesResponse, MetadataItem, updateSample, ValidateSampleNameModel, @@ -31,6 +33,7 @@ export interface MetadataHeaderItem { interface MetadataValidateDetailsItem { isSampleNameValid: boolean; foundSampleId?: number; + locked: boolean; } interface MetadataSaveDetailsItem { @@ -187,22 +190,29 @@ export const setSampleNameColumn = createAsyncThunk< .map((row) => ({ name: row[updatedSampleNameColumn], })); - const response: ValidateSamplesResponse = await validateSamples({ + const validatedSamples: ValidateSamplesResponse = await validateSamples({ projectId: projectId, body: { samples: samples, }, }); + const lockedSamples: LockedSamplesResponse = await getLockedSamples({ + projectId, + }); for (const metadataItem of metadata) { const index: string = metadataItem.rowKey; const sampleName: string = metadataItem[updatedSampleNameColumn]; - const foundSample: ValidateSampleNameModel | undefined = - response.samples.find( - (sample: ValidateSampleNameModel) => sampleName === sample.name - ); + const foundValidatedSamples = validatedSamples.samples.find( + (sample) => sampleName === sample.name + ); + const foundSampleId = foundValidatedSamples?.ids?.at(0); + const foundLockedSamples = lockedSamples.sampleIds.find( + (sampleId) => sampleId === foundSampleId + ); metadataValidateDetails[index] = { isSampleNameValid: validateSampleName(sampleName), - foundSampleId: foundSample?.ids?.at(0), + foundSampleId: foundSampleId, + locked: !!foundLockedSamples, }; } From bde9fc62515ca5987d73dce4dd2e85f5d07e7e8a Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 21 Nov 2022 05:28:54 -0600 Subject: [PATCH 417/655] chore: Added sample and project identifiers to the table --- src/main/resources/i18n/messages.properties | 1 + .../resources/js/pages/search/SearchProjectsTable.tsx | 7 +++++++ .../resources/js/pages/search/SearchSamplesTable.tsx | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index 007c3b976c6..b66b5f032ca 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2183,6 +2183,7 @@ SearchLayout.projects=Projects SearchLayout.samples=Samples SearchLayout.admin.global=Everywhere SearchLayout.admin.personal=Personal +SearchTable.identifier=ID SearchTable.name=Name SearchTable.organism=Organism SearchTable.created=Create diff --git a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx index 1899fb020ab..bdc78489842 100644 --- a/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchProjectsTable.tsx @@ -28,6 +28,12 @@ export default function SearchProjectsTable({ }: SearchProjectsTableParams) { const columns = useMemo>( () => [ + { + key: `identifier`, + dataIndex: "id", + title: i18n("SearchTable.identifier"), + width: 100, + }, { key: `name`, dataIndex: "name", @@ -70,6 +76,7 @@ export default function SearchProjectsTable({ return (
    >( () => [ + { + key: `identifier`, + dataIndex: "id", + title: i18n("SearchTable.identifier"), + }, { key: `sampleName`, dataIndex: "name", From 07b5a3cb576df434b5b30c1374a6ef8bb8668f55 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Mon, 21 Nov 2022 11:28:03 -0600 Subject: [PATCH 418/655] adding locked sample column to review table --- .../js/components/files/DragUpload.tsx | 4 ---- .../components/SampleMetadataImportReview.tsx | 23 +++++++++++++++---- .../{SampleIcons.jsx => SampleIcons.tsx} | 11 +++++++-- 3 files changed, 28 insertions(+), 10 deletions(-) rename src/main/webapp/resources/js/pages/projects/samples/components/{SampleIcons.jsx => SampleIcons.tsx} (77%) diff --git a/src/main/webapp/resources/js/components/files/DragUpload.tsx b/src/main/webapp/resources/js/components/files/DragUpload.tsx index 08e365defd7..97d6b498070 100644 --- a/src/main/webapp/resources/js/components/files/DragUpload.tsx +++ b/src/main/webapp/resources/js/components/files/DragUpload.tsx @@ -18,10 +18,6 @@ export interface DragUploadProps { } /** * React component for rendering the drag and drop upload functionality. - * @param uploadText - text for drag and drop - * @param uploadHint - hint for drag and drop - * @param options - upload options - * @param props - remainder of props passed * @returns {*} * @constructor */ diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx index ba7ce206aac..7b4db286bc6 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx @@ -28,6 +28,7 @@ import { import { MetadataItem } from "../../../../apis/projects/samples"; import { ColumnsType, ColumnType } from "antd/es/table"; import { TableRowSelection } from "antd/lib/table/interface"; +import SampleIcons from "../../samples/components/SampleIcons"; const { Paragraph, Text } = Typography; @@ -83,8 +84,9 @@ export function SampleMetadataImportReview(): JSX.Element { }, getCheckboxProps: (record: MetadataItem) => ({ disabled: !( - metadataValidateDetails[record.rowKey].isSampleNameValid || - metadataSaveDetails[record.rowKey]?.saved === true + (metadataValidateDetails[record.rowKey].isSampleNameValid && + !metadataValidateDetails[record.rowKey].locked) || + metadataSaveDetails[record.rowKey]?.saved ), }), }; @@ -103,6 +105,17 @@ export function SampleMetadataImportReview(): JSX.Element { ) ); + const lockedColumn: ColumnType = { + title: "", + dataIndex: "locked", + fixed: "left", + width: 100, + render: (text, item) => { + const sample = { owner: !metadataValidateDetails[item.rowKey].locked }; + return ; + }, + }; + const sampleColumn: ColumnType = { title: sampleNameColumn, dataIndex: sampleNameColumn, @@ -177,6 +190,7 @@ export function SampleMetadataImportReview(): JSX.Element { const updatedColumns: ColumnsType = [ savedColumn, + lockedColumn, sampleColumn, tagColumn, ...otherColumns, @@ -187,8 +201,9 @@ export function SampleMetadataImportReview(): JSX.Element { metadata .filter( (row) => - metadataValidateDetails[row.rowKey].isSampleNameValid || - metadataSaveDetails[row.rowKey]?.saved === true + (metadataValidateDetails[row.rowKey].isSampleNameValid && + !metadataValidateDetails[row.rowKey]?.locked) || + metadataSaveDetails[row.rowKey]?.saved ) .map((row): string => row.rowKey) ); diff --git a/src/main/webapp/resources/js/pages/projects/samples/components/SampleIcons.jsx b/src/main/webapp/resources/js/pages/projects/samples/components/SampleIcons.tsx similarity index 77% rename from src/main/webapp/resources/js/pages/projects/samples/components/SampleIcons.jsx rename to src/main/webapp/resources/js/pages/projects/samples/components/SampleIcons.tsx index 52eb3c254f2..335e5e658d2 100644 --- a/src/main/webapp/resources/js/pages/projects/samples/components/SampleIcons.jsx +++ b/src/main/webapp/resources/js/pages/projects/samples/components/SampleIcons.tsx @@ -3,14 +3,21 @@ import { LockTwoTone } from "@ant-design/icons"; import { Popover, Space } from "antd"; import { red6 } from "../../../../styles/colors"; +interface Sample { + owner: boolean; +} + +export interface SampleIconsProps { + sample: Sample; +} + /** * React component to render any icons onto the sample listing table that * give extra information about the sample. - * @param {object} sample * @returns {JSX.Element} * @constructor */ -export default function SampleIcons({ sample }) { +export default function SampleIcons({ sample }: SampleIconsProps): JSX.Element { return ( {!sample.owner && ( From 75b448856bb9fa0a4e8dde7ead6a31df75f704a7 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 21 Nov 2022 13:16:35 -0600 Subject: [PATCH 419/655] fix: Fix width of ID column --- .../webapp/resources/js/pages/search/SearchSamplesTable.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx index 3832d638d2d..ffd81659ada 100644 --- a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -33,6 +33,7 @@ export default function SearchSamplesTable({ key: `identifier`, dataIndex: "id", title: i18n("SearchTable.identifier"), + width: 100, }, { key: `sampleName`, @@ -45,7 +46,7 @@ export default function SearchSamplesTable({ sampleId={sample.id} projectId={sample.projects[0].id} > - + ); }, From 2bf3e136d00aecbe61dc10d3d6358b0071e017e2 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Mon, 21 Nov 2022 15:15:32 -0600 Subject: [PATCH 420/655] moving locked samples into an alert --- src/main/resources/i18n/messages.properties | 12 +-- .../alerts/{ErrorAlert.jsx => ErrorAlert.tsx} | 14 +-- .../components/SampleMetadataImportReview.tsx | 89 ++++++++++++------- 3 files changed, 70 insertions(+), 45 deletions(-) rename src/main/webapp/resources/js/components/alerts/{ErrorAlert.jsx => ErrorAlert.tsx} (55%) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index 43e5b30df6f..6e65d19cfc6 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2010,11 +2010,13 @@ SampleMetadataImportReview.tab.found=Samples to be updated SampleMetadataImportReview.tab.missing=Samples to be created SampleMetadataImportReview.table.filter.new=New SampleMetadataImportReview.table.filter.existing=Existing -SampleMetadataImportReview.alert.title=Validation Error -SampleMetadataImportReview.alert.description=Please correct the following errors within the file and re-upload. The sample name must meet the following criteria: -SampleMetadataImportReview.alert.rule1=cannot be empty -SampleMetadataImportReview.alert.rule2=minimum 3 characters long -SampleMetadataImportReview.alert.rule3=contain only alphanumeric characters and '-', '_' +SampleMetadataImportReview.alert.valid.title=Validation Error +SampleMetadataImportReview.alert.valid.description=Please correct the following errors within the file and re-upload. The sample name must meet the following criteria: +SampleMetadataImportReview.alert.valid.rule1=cannot be empty +SampleMetadataImportReview.alert.valid.rule2=minimum 3 characters long +SampleMetadataImportReview.alert.valid.rule3=contain only alphanumeric characters and '-', '_' +SampleMetadataImportReview.alert.locked.description.popover.content={0} sample metadata +SampleMetadataImportReview.alert.locked.description=\ cannot be imported, because these samples are locked in the current project. SampleMetadataImportComplete.result.title=The sample metadata imported successfully! SampleMetadataImportComplete.button.upload=Upload another file diff --git a/src/main/webapp/resources/js/components/alerts/ErrorAlert.jsx b/src/main/webapp/resources/js/components/alerts/ErrorAlert.tsx similarity index 55% rename from src/main/webapp/resources/js/components/alerts/ErrorAlert.jsx rename to src/main/webapp/resources/js/components/alerts/ErrorAlert.tsx index 83671ddacc7..4bd7dee8264 100644 --- a/src/main/webapp/resources/js/components/alerts/ErrorAlert.jsx +++ b/src/main/webapp/resources/js/components/alerts/ErrorAlert.tsx @@ -4,16 +4,18 @@ import React from "react"; import { Alert } from "antd"; +import { AlertProps } from "antd/lib/alert"; /** * Stateless UI component for displaying an [antd error Alert]{@link https://ant.design/components/alert/} * - * @param {string} message - Text to display in alert - * @param {string} description - Optional description - * @param {object} props - any other props that are passed - * @returns {Element} - Returns an antd error 'Alert' component + * @returns {JSX.Element} - Returns an antd error 'Alert' component */ -export function ErrorAlert({ message, description, ...props }) { +export function ErrorAlert({ + message, + description, + ...props +}: AlertProps): JSX.Element { return ( ); -} \ No newline at end of file +} diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx index 7b4db286bc6..d007a839496 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx @@ -3,6 +3,8 @@ import { useNavigate, useParams } from "react-router-dom"; import { Alert, Button, + List, + Popover, Table, TableProps, Tag, @@ -28,7 +30,7 @@ import { import { MetadataItem } from "../../../../apis/projects/samples"; import { ColumnsType, ColumnType } from "antd/es/table"; import { TableRowSelection } from "antd/lib/table/interface"; -import SampleIcons from "../../samples/components/SampleIcons"; +import { ErrorAlert } from "../../../../../js/components/alerts/ErrorAlert"; const { Paragraph, Text } = Typography; @@ -64,7 +66,6 @@ export function SampleMetadataImportReview(): JSX.Element { const navigate: NavigateFunction = useNavigate(); const [columns, setColumns] = React.useState>([]); const [selected, setSelected] = React.useState([]); - const [valid, setValid] = React.useState(true); const [progress, setProgress] = React.useState(0); const [loading, setLoading] = React.useState(false); const { @@ -84,8 +85,7 @@ export function SampleMetadataImportReview(): JSX.Element { }, getCheckboxProps: (record: MetadataItem) => ({ disabled: !( - (metadataValidateDetails[record.rowKey].isSampleNameValid && - !metadataValidateDetails[record.rowKey].locked) || + metadataValidateDetails[record.rowKey].isSampleNameValid || metadataSaveDetails[record.rowKey]?.saved ), }), @@ -99,23 +99,6 @@ export function SampleMetadataImportReview(): JSX.Element { }, [metadataSaveDetails, selected.length]); React.useEffect(() => { - setValid( - !metadata.some( - (row) => !metadataValidateDetails[row.rowKey].isSampleNameValid - ) - ); - - const lockedColumn: ColumnType = { - title: "", - dataIndex: "locked", - fixed: "left", - width: 100, - render: (text, item) => { - const sample = { owner: !metadataValidateDetails[item.rowKey].locked }; - return ; - }, - }; - const sampleColumn: ColumnType = { title: sampleNameColumn, dataIndex: sampleNameColumn, @@ -190,7 +173,6 @@ export function SampleMetadataImportReview(): JSX.Element { const updatedColumns: ColumnsType = [ savedColumn, - lockedColumn, sampleColumn, tagColumn, ...otherColumns, @@ -201,8 +183,7 @@ export function SampleMetadataImportReview(): JSX.Element { metadata .filter( (row) => - (metadataValidateDetails[row.rowKey].isSampleNameValid && - !metadataValidateDetails[row.rowKey]?.locked) || + metadataValidateDetails[row.rowKey].isSampleNameValid || metadataSaveDetails[row.rowKey]?.saved ) .map((row): string => row.rowKey) @@ -238,23 +219,62 @@ export function SampleMetadataImportReview(): JSX.Element { } }; + const isValid = !metadata.some( + (row) => !metadataValidateDetails[row.rowKey].isSampleNameValid + ); + + const lockedSampleMetadata = metadata.filter( + (metadataItem) => metadataValidateDetails[metadataItem.rowKey].locked + ); + return ( {i18n("SampleMetadataImportReview.description")} - {!valid && ( - - {i18n("SampleMetadataImportReview.alert.description")} + {i18n("SampleMetadataImportReview.alert.valid.description")}
      -
    • {i18n("SampleMetadataImportReview.alert.rule1")}
      • -
    • {i18n("SampleMetadataImportReview.alert.rule2")}
      • -
    • {i18n("SampleMetadataImportReview.alert.rule3")}
      • +
    • {i18n("SampleMetadataImportReview.alert.valid.rule1")}
      • +
    • {i18n("SampleMetadataImportReview.alert.valid.rule2")}
      • +
    • {i18n("SampleMetadataImportReview.alert.valid.rule3")}
    } - type="error" + /> + )} + {lockedSampleMetadata.length > 0 && ( + + + ( + {metadataItem[sampleNameColumn]} + )} + /> + + } + > + + {i18n( + "SampleMetadataImportReview.alert.locked.description.popover.content", + lockedSampleMetadata.length + )} + + + {i18n("SampleMetadataImportReview.alert.locked.description")} + + } + type="warning" showIcon /> )} @@ -266,10 +286,11 @@ export function SampleMetadataImportReview(): JSX.Element { } rowSelection={rowSelection} columns={columns} - dataSource={metadata} + dataSource={metadata.filter( + (metadataItem) => !metadataValidateDetails[metadataItem.rowKey].locked + )} pagination={getPaginationOptions(metadata.length)} /> -
    @@ -303,7 +305,7 @@ export function SampleMetadataImportReview(): JSX.Element { className="t-metadata-uploader-upload-button" style={{ marginLeft: "auto" }} onClick={save} - loading={loading} + disabled={loading} > {i18n("SampleMetadataImportReview.button.next")} From 6e72446cda0bab3f90f727f164ae28ab3c06b828 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Fri, 25 Nov 2022 11:07:17 -0600 Subject: [PATCH 431/655] adding promise chaining back in --- .../redux/importReducer.ts | 106 +++++++++++------- 1 file changed, 65 insertions(+), 41 deletions(-) diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts index d2ed85a2024..07e40791f74 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/redux/importReducer.ts @@ -102,8 +102,9 @@ export const saveMetadata = createAsyncThunk< ); }); - const chunkSize = 100; + const chunkSize = 10; + const updateSamplesPromises: Promise[] = []; const updateSampleList = selectedSampleList .filter((metadataItem) => { const name = metadataItem[sampleNameColumn]; @@ -127,34 +128,46 @@ export const saveMetadata = createAsyncThunk< })); return { name, sampleId, metadata: metadataFields }; }); - if (updateSampleList.length > 0) { const chunkedUpdateSampleList = chunkArray(updateSampleList, chunkSize); for (const chunk of chunkedUpdateSampleList) { - await updateSamples({ - projectId, - body: chunk, - }) - .then(() => { - chunk - .map((item) => item.name) - .forEach((name) => (metadataSaveDetails[name] = { saved: true })); + updateSamplesPromises.push( + updateSamples({ + projectId, + body: chunk, }) - .catch((error) => { - const { errors } = error.response.data; - Object.keys(errors).map((key) => { - metadataSaveDetails[key] = { - saved: false, - error: errors[key], - }; - }); - }); - dispatch( - setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) + .then(() => { + chunk + .map((item) => item.name) + .forEach( + (name) => (metadataSaveDetails[name] = { saved: true }) + ); + }) + .catch((error) => { + const { errors } = error.response.data; + Object.keys(errors).map((key) => { + metadataSaveDetails[key] = { + saved: false, + error: errors[key], + }; + }); + }) ); } + const chunkedUpdateSamplesPromises = chunkArray( + updateSamplesPromises, + chunkSize + ); + for (const chunk of chunkedUpdateSamplesPromises) { + await Promise.all(chunk).then(() => { + dispatch( + setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) + ); + }); + } } + const createSamplesPromises: Promise[] = []; const createSampleList = selectedSampleList .filter((metadataItem) => { const name = metadataItem[sampleNameColumn]; @@ -181,29 +194,40 @@ export const saveMetadata = createAsyncThunk< if (createSampleList.length > 0) { const chunkedCreateSampleList = chunkArray(createSampleList, chunkSize); for (const chunk of chunkedCreateSampleList) { - await createSamples({ - projectId, - body: chunk, - }) - .then(() => { - chunk - .map((item) => item.name) - .forEach((name) => (metadataSaveDetails[name] = { saved: true })); + createSamplesPromises.push( + createSamples({ + projectId, + body: chunk, }) - .catch((error) => { - const { errors } = error.response.data; - Object.keys(errors).map((key) => { - metadataSaveDetails[key] = { - saved: false, - error: errors[key], - }; - }); - }); - - dispatch( - setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) + .then(() => { + chunk + .map((item) => item.name) + .forEach( + (name) => (metadataSaveDetails[name] = { saved: true }) + ); + }) + .catch((error) => { + const { errors } = error.response.data; + Object.keys(errors).map((key) => { + metadataSaveDetails[key] = { + saved: false, + error: errors[key], + }; + }); + }) ); } + const chunkedCreateSamplesPromises = chunkArray( + createSamplesPromises, + chunkSize + ); + for (const chunk of chunkedCreateSamplesPromises) { + await Promise.all(chunk).then(() => { + dispatch( + setMetadataSaveDetails(Object.assign({}, metadataSaveDetails)) + ); + }); + } } return { metadataSaveDetails }; From 9e816f766d5083b031602ea89abe712e783a11ed Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 28 Nov 2022 09:11:36 -0600 Subject: [PATCH 432/655] chore: API for search --- .../webapp/resources/js/apis/search/search.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/main/webapp/resources/js/apis/search/search.ts diff --git a/src/main/webapp/resources/js/apis/search/search.ts b/src/main/webapp/resources/js/apis/search/search.ts new file mode 100644 index 00000000000..cc4ef2755bf --- /dev/null +++ b/src/main/webapp/resources/js/apis/search/search.ts @@ -0,0 +1,62 @@ +import axios from "axios"; +import { setBaseUrl } from "../../utilities/url-utilities"; +import { TablePaginationConfig } from "antd"; + +export type SearchParams = { + global: boolean; + pagination: TablePaginationConfig | undefined; + order: [ + { + property: string; + direction: "asc" | "desc"; + } + ]; + query: string | null; +}; + +/** + * Global search for projects + * @param params + */ +async function fetchSearchSamples(params: SearchParams) { + return axios.post(setBaseUrl(`/ajax/search/samples`), { + global, + pagination: params.pagination, + order: params.order, + search: [ + { + property: `name`, + value: params.query, + operation: "MATCH_IN", + }, + ], + }); +} + +/** + * Global search for samples + * @param params + */ +async function fetchSearchProjects(params: SearchParams) { + return axios.post(setBaseUrl(`/ajax/search/projects`), { + global, + pagination: params.pagination, + order: params.order, + search: [ + { + property: `name`, + value: params.query, + operation: "MATCH_IN", + }, + ], + }); +} + +export async function fetchSearchResults(params: SearchParams) { + const promises = []; + promises.push(fetchSearchProjects(params)); + promises.push(fetchSearchSamples(params)); + return Promise.all(promises).then( + ([{ data: projects }, { data: samples }]) => ({ projects, samples }) + ); +} From 89edf7e257bb0660eb890eda2d8c7e7b2cce9222 Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 28 Nov 2022 11:29:26 -0600 Subject: [PATCH 433/655] chore: Updated API for search --- .../webapp/resources/js/apis/search/search.ts | 55 +++++-------- .../js/pages/search/SearchLayout.tsx | 81 ++++++++++--------- 2 files changed, 60 insertions(+), 76 deletions(-) diff --git a/src/main/webapp/resources/js/apis/search/search.ts b/src/main/webapp/resources/js/apis/search/search.ts index cc4ef2755bf..a93cf0f95aa 100644 --- a/src/main/webapp/resources/js/apis/search/search.ts +++ b/src/main/webapp/resources/js/apis/search/search.ts @@ -1,6 +1,7 @@ -import axios from "axios"; +import axios, { AxiosResponse } from "axios"; import { setBaseUrl } from "../../utilities/url-utilities"; import { TablePaginationConfig } from "antd"; +import { SearchProject, SearchSample } from "../../pages/search/SearchLayout"; export type SearchParams = { global: boolean; @@ -11,52 +12,34 @@ export type SearchParams = { direction: "asc" | "desc"; } ]; - query: string | null; + search: [ + { + property: string; + value: string; + operation: "MATCH_IN"; + } + ]; }; /** * Global search for projects * @param params */ -async function fetchSearchSamples(params: SearchParams) { - return axios.post(setBaseUrl(`/ajax/search/samples`), { - global, - pagination: params.pagination, - order: params.order, - search: [ - { - property: `name`, - value: params.query, - operation: "MATCH_IN", - }, - ], - }); +export async function fetchSearchSamples( + params: SearchParams +): Promise> { + return axios.post(setBaseUrl(`/ajax/search/samples`), params); } /** * Global search for samples * @param params */ -async function fetchSearchProjects(params: SearchParams) { - return axios.post(setBaseUrl(`/ajax/search/projects`), { - global, - pagination: params.pagination, - order: params.order, - search: [ - { - property: `name`, - value: params.query, - operation: "MATCH_IN", - }, - ], - }); -} - -export async function fetchSearchResults(params: SearchParams) { - const promises = []; - promises.push(fetchSearchProjects(params)); - promises.push(fetchSearchSamples(params)); - return Promise.all(promises).then( - ([{ data: projects }, { data: samples }]) => ({ projects, samples }) +export async function fetchSearchProjects( + params: SearchParams +): Promise> { + return await axios.post( + setBaseUrl(`/ajax/search/projects`), + params ); } diff --git a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx index 4c4b7fff7a8..5f05315c3cf 100644 --- a/src/main/webapp/resources/js/pages/search/SearchLayout.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchLayout.tsx @@ -14,11 +14,14 @@ import { Typography, } from "antd"; import { FilterValue } from "antd/es/table/interface"; -import axios from "axios"; -import { setBaseUrl } from "../../utilities/url-utilities"; import type { CurrentUser, Sample } from "../../types/irida"; import { debounce } from "lodash"; import SearchCount from "./SearchCount"; +import { + fetchSearchProjects, + fetchSearchSamples, + SearchParams, +} from "../../apis/search/search"; type SearchType = "projects" | "samples"; type SearchItem = { @@ -120,47 +123,45 @@ export default function SearchLayout() { }; const fetchData = useCallback(() => { - const fetchSamples = async () => - axios.post(setBaseUrl(`/ajax/search/samples`), { - global, - pagination: samplesTableParams.pagination, - order: [ - { - property: samplesTableParams.columnKey || `sampleName`, - direction: samplesTableParams.order === "ascend" ? `asc` : `desc`, - }, - ], - search: [ - { - property: `name`, - value: searchParams.get("query"), - operation: "MATCH_IN", - }, - ], - }); + const sampleParams: SearchParams = { + global, + pagination: samplesTableParams.pagination, + order: [ + { + property: samplesTableParams.columnKey || `sampleName`, + direction: samplesTableParams.order === "ascend" ? `asc` : `desc`, + }, + ], + search: [ + { + property: `name`, + value: searchParams.get("query") || "", + operation: "MATCH_IN", + }, + ], + }; - const fetchProjects = async () => - axios.post(setBaseUrl(`/ajax/search/projects`), { - global, - pagination: projectsTableParams.pagination, - order: [ - { - property: projectsTableParams.columnKey || `name`, - direction: projectsTableParams.order === "ascend" ? `asc` : `desc`, - }, - ], - search: [ - { - property: `name`, - value: searchParams.get("query"), - operation: "MATCH_IN", - }, - ], - }); + const projectParams: SearchParams = { + global, + pagination: projectsTableParams.pagination, + order: [ + { + property: projectsTableParams.columnKey || `name`, + direction: projectsTableParams.order === "ascend" ? `asc` : `desc`, + }, + ], + search: [ + { + property: `name`, + value: searchParams.get("query") || "", + operation: "MATCH_IN", + }, + ], + }; const promises = []; - promises.push(fetchProjects()); - promises.push(fetchSamples()); + promises.push(fetchSearchProjects(projectParams)); + promises.push(fetchSearchSamples(sampleParams)); Promise.all(promises).then( ([{ data: projectsData }, { data: samplesData }]) => { setProjects(projectsData); From 22954f224c8b19351706000e24d3a8e55a76104f Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 28 Nov 2022 12:02:36 -0600 Subject: [PATCH 434/655] refactor: Used styled-components to clean up repeated styles --- .../resources/js/pages/search/SearchCount.tsx | 6 ++- .../js/pages/search/SearchLayout.tsx | 38 ++++++++----------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/main/webapp/resources/js/pages/search/SearchCount.tsx b/src/main/webapp/resources/js/pages/search/SearchCount.tsx index 55085da30f2..bf6c12849e2 100644 --- a/src/main/webapp/resources/js/pages/search/SearchCount.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchCount.tsx @@ -6,7 +6,11 @@ import { Badge } from "antd"; * @param count * @constructor */ -export default function SearchCount({ count }: { count: number }) { +export default function SearchCount({ + count = 0, +}: { + count: number | undefined; +}) { return ( [ { label: ( -
    - {i18n("SearchLayout.projects").toUpperCase()} - -
    + ), key: "projects", }, { label: ( -
    - {i18n("SearchLayout.samples").toUpperCase()} - -
    + ), key: "samples", }, From 68379c858a9a0e678a7d7d4f64340f14e6ba992b Mon Sep 17 00:00:00 2001 From: Josh Adam Date: Mon, 28 Nov 2022 12:04:53 -0600 Subject: [PATCH 435/655] refactor: Made the sample details viewer button a link --- .../webapp/resources/js/pages/search/SearchSamplesTable.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx index ffd81659ada..9788ee059ca 100644 --- a/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx +++ b/src/main/webapp/resources/js/pages/search/SearchSamplesTable.tsx @@ -46,7 +46,9 @@ export default function SearchSamplesTable({ sampleId={sample.id} projectId={sample.projects[0].id} > - + ); }, From bcb14d0c531e27ab810d61f629751c943106a739 Mon Sep 17 00:00:00 2001 From: Katherine Thiessen Date: Thu, 1 Dec 2022 11:58:47 -0600 Subject: [PATCH 436/655] replacing step progress bar with table overlay progress bar --- src/main/resources/i18n/messages.properties | 1 + .../components/SampleMetadataImportReview.tsx | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/resources/i18n/messages.properties b/src/main/resources/i18n/messages.properties index 6e65d19cfc6..5d89aa77edb 100644 --- a/src/main/resources/i18n/messages.properties +++ b/src/main/resources/i18n/messages.properties @@ -2004,6 +2004,7 @@ SampleMetadataImportMapColumns.table.existingRestriction=Existing Restriction SampleMetadataImportMapColumns.table.targetRestriction=Target Restriction SampleMetadataImportReview.description=Review the metadata to be uploaded. +SampleMetadataImportReview.loading=Please wait until the upload completes. Closing this window or leaving this page will terminate the upload. SampleMetadataImportReview.button.back=Select a different column SampleMetadataImportReview.button.next=Upload the data SampleMetadataImportReview.tab.found=Samples to be updated diff --git a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx index f22bf937c13..6ab196a1f2c 100644 --- a/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx +++ b/src/main/webapp/resources/js/pages/projects/samples-metadata-import/components/SampleMetadataImportReview.tsx @@ -5,6 +5,7 @@ import { Button, List, Popover, + Progress, Table, TableProps, Tag, @@ -228,8 +229,12 @@ export function SampleMetadataImportReview(): JSX.Element { ); return ( - - {i18n("SampleMetadataImportReview.description")} + + + {loading + ? i18n("SampleMetadataImportReview.loading") + : i18n("SampleMetadataImportReview.description")} + {!isValid && ( !metadataValidateDetails[metadataItem.rowKey].locked )} pagination={getPaginationOptions(metadata.length)} - loading={loading} + loading={{ + indicator: , + spinning: loading, + }} />