From d2c8336bca448fb73754c9879566e6d31c549169 Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Thu, 12 Sep 2024 09:32:37 -0500 Subject: [PATCH] Updated command line executor and kubernetes executor based on changes to IN-CORE Dataset DAO and file storage --- .../commandline/CommandLineExecutor.java | 19 +++++++++++++------ .../kubernetes/KubernetesExecutor.java | 16 +++++++++++----- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java b/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java index 27189e16..aa4e63c6 100644 --- a/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java +++ b/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java @@ -173,6 +173,12 @@ public void execute(File cwd) throws AbortException, FailedException { // Create a folder for the datasets File inputFolder = new File(filename); + if (inputFolder.exists()) { + // For single file, a tmp file got created above; however in this case, we need + // a temporary folder to store the files + inputFolder.delete(); + } + if (!inputFolder.mkdirs()) { throw (new FailedException("Could not create folder for input files")); } @@ -251,6 +257,7 @@ public void execute(File cwd) throws AbortException, FailedException { sb.append(" "); } println("Executing : " + sb.toString()); + logger.debug("Executing : " + sb.toString()); // create the process builder ProcessBuilder pb = new ProcessBuilder(command); @@ -369,11 +376,11 @@ public void execute(File cwd) throws AbortException, FailedException { ds.setTitle(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle()); ds.setCreator(execution.getCreator()); + ds = datasetDao.save(ds); + ByteArrayInputStream bais = new ByteArrayInputStream(stdout.toString().getBytes("UTF-8")); FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle(), bais, execution.getCreator(), ds); - ds = datasetDao.save(ds); - execution.setDataset(step.getOutputs().get(impl.getCaptureStdOut()), ds.getId()); saveExecution = true; } catch (IOException exc) { @@ -385,11 +392,11 @@ public void execute(File cwd) throws AbortException, FailedException { Dataset ds = new Dataset(); ds.setTitle(step.getTool().getOutput(impl.getCaptureStdErr()).getTitle()); ds.setCreator(execution.getCreator()); + ds = datasetDao.save(ds); ByteArrayInputStream bais = new ByteArrayInputStream(stderr.toString().getBytes("UTF-8")); FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdErr()).getTitle(), bais, execution.getCreator(), ds); - ds = datasetDao.save(ds); execution.setDataset(step.getOutputs().get(impl.getCaptureStdErr()), ds.getId()); saveExecution = true; @@ -419,15 +426,15 @@ public boolean accept(File pathname) { for (File file : files) { logger.debug("adding files to a dataset: " + file); FileInputStream fis = new FileInputStream(file); - fileStorage.storeFile(file.getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(file.getName(), fis, execution.getCreator(), ds); fis.close(); } } else { FileInputStream fis = new FileInputStream(entry.getValue()); - fileStorage.storeFile(new File(entry.getValue()).getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(new File(entry.getValue()).getName(), fis, execution.getCreator(), ds); } - ds = datasetDao.save(ds); +// ds = datasetDao.save(ds); execution.setDataset(step.getOutputs().get(entry.getKey()), ds.getId()); saveExecution = true; diff --git a/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java b/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java index 2b36aaf8..c366c8a4 100644 --- a/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java +++ b/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java @@ -159,10 +159,17 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException { // Create a folder for the datasets File inputFolder = new File(filename); + if (inputFolder.exists()) { + // For single file, a tmp file got created above; however in this case, we need + // a temporary folder to store the files + inputFolder.delete(); + } + if (!inputFolder.mkdirs()) { throw (new FailedException("Could not create folder for input files")); } + int duplicate = 1; for (FileDescriptor fd : ds.getFileDescriptors()) { String localFileName = fd.getFilename(); @@ -384,12 +391,11 @@ public State checkRemoteJob() throws FailedException { Dataset ds = new Dataset(); ds.setTitle(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle()); ds.setCreator(execution.getCreator()); + ds = datasetDao.save(ds); ByteArrayInputStream bais = new ByteArrayInputStream(lastlog.getBytes("UTF-8")); FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle(), bais, execution.getCreator(), ds); - ds = datasetDao.save(ds); - execution.setDataset(step.getOutputs().get(impl.getCaptureStdOut()), ds.getId()); saveExecution = true; } @@ -419,15 +425,15 @@ public boolean accept(File pathname) { for (File file : files) { logger.debug("adding files to a dataset: " + file); FileInputStream fis = new FileInputStream(file); - fileStorage.storeFile(file.getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(file.getName(), fis, execution.getCreator(), ds); fis.close(); } } else { FileInputStream fis = new FileInputStream(entry.getValue()); - fileStorage.storeFile(new File(entry.getValue()).getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(new File(entry.getValue()).getName(), fis, execution.getCreator(), ds); } - ds = datasetDao.save(ds); +// ds = datasetDao.save(ds); execution.setDataset(step.getOutputs().get(entry.getKey()), ds.getId()); saveExecution = true;