Skip to content

Commit

Permalink
Updated command line executor and kubernetes executor based on change…
Browse files Browse the repository at this point in the history 
…s to IN-CORE Dataset DAO and file storage
  • Loading branch information
@navarroc
navarroc committed Sep 12, 2024
1 parent 94ce8e6 commit d2c8336
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 11 deletions.
19 changes: 13 additions & 6 deletions ...ne/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,12 @@ public void execute(File cwd) throws AbortException, FailedException {

// Create a folder for the datasets
File inputFolder = new File(filename);
if (inputFolder.exists()) {
// For single file, a tmp file got created above; however in this case, we need
// a temporary folder to store the files
inputFolder.delete();
}

if (!inputFolder.mkdirs()) {
throw (new FailedException("Could not create folder for input files"));
}
Expand Down Expand Up @@ -251,6 +257,7 @@ public void execute(File cwd) throws AbortException, FailedException {
sb.append(" ");
}
println("Executing : " + sb.toString());
logger.debug("Executing : " + sb.toString());

// create the process builder
ProcessBuilder pb = new ProcessBuilder(command);
Expand Down Expand Up @@ -369,11 +376,11 @@ public void execute(File cwd) throws AbortException, FailedException {
ds.setTitle(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle());
ds.setCreator(execution.getCreator());

ds = datasetDao.save(ds);

ByteArrayInputStream bais = new ByteArrayInputStream(stdout.toString().getBytes("UTF-8"));
FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle(), bais, execution.getCreator(), ds);

ds = datasetDao.save(ds);

execution.setDataset(step.getOutputs().get(impl.getCaptureStdOut()), ds.getId());
saveExecution = true;
} catch (IOException exc) {
Expand All @@ -385,11 +392,11 @@ public void execute(File cwd) throws AbortException, FailedException {
Dataset ds = new Dataset();
ds.setTitle(step.getTool().getOutput(impl.getCaptureStdErr()).getTitle());
ds.setCreator(execution.getCreator());
ds = datasetDao.save(ds);

ByteArrayInputStream bais = new ByteArrayInputStream(stderr.toString().getBytes("UTF-8"));
FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdErr()).getTitle(), bais, execution.getCreator(), ds);

ds = datasetDao.save(ds);

execution.setDataset(step.getOutputs().get(impl.getCaptureStdErr()), ds.getId());
saveExecution = true;
Expand Down Expand Up @@ -419,15 +426,15 @@ public boolean accept(File pathname) {
for (File file : files) {
logger.debug("adding files to a dataset: " + file);
FileInputStream fis = new FileInputStream(file);
fileStorage.storeFile(file.getName(), fis, ds.getCreator(), ds);
fileStorage.storeFile(file.getName(), fis, execution.getCreator(), ds);
fis.close();
}

} else {
FileInputStream fis = new FileInputStream(entry.getValue());
fileStorage.storeFile(new File(entry.getValue()).getName(), fis, ds.getCreator(), ds);
fileStorage.storeFile(new File(entry.getValue()).getName(), fis, execution.getCreator(), ds);
}
ds = datasetDao.save(ds);
// ds = datasetDao.save(ds);

execution.setDataset(step.getOutputs().get(entry.getKey()), ds.getId());
saveExecution = true;
Expand Down
16 changes: 11 additions & 5 deletions ...etes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,17 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException {

// Create a folder for the datasets
File inputFolder = new File(filename);
if (inputFolder.exists()) {
// For single file, a tmp file got created above; however in this case, we need
// a temporary folder to store the files
inputFolder.delete();
}

if (!inputFolder.mkdirs()) {
throw (new FailedException("Could not create folder for input files"));
}


int duplicate = 1;
for (FileDescriptor fd : ds.getFileDescriptors()) {
String localFileName = fd.getFilename();
Expand Down Expand Up @@ -384,12 +391,11 @@ public State checkRemoteJob() throws FailedException {
Dataset ds = new Dataset();
ds.setTitle(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle());
ds.setCreator(execution.getCreator());
ds = datasetDao.save(ds);

ByteArrayInputStream bais = new ByteArrayInputStream(lastlog.getBytes("UTF-8"));
FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle(), bais, execution.getCreator(), ds);

ds = datasetDao.save(ds);

execution.setDataset(step.getOutputs().get(impl.getCaptureStdOut()), ds.getId());
saveExecution = true;
}
Expand Down Expand Up @@ -419,15 +425,15 @@ public boolean accept(File pathname) {
for (File file : files) {
logger.debug("adding files to a dataset: " + file);
FileInputStream fis = new FileInputStream(file);
fileStorage.storeFile(file.getName(), fis, ds.getCreator(), ds);
fileStorage.storeFile(file.getName(), fis, execution.getCreator(), ds);
fis.close();
}

} else {
FileInputStream fis = new FileInputStream(entry.getValue());
fileStorage.storeFile(new File(entry.getValue()).getName(), fis, ds.getCreator(), ds);
fileStorage.storeFile(new File(entry.getValue()).getName(), fis, execution.getCreator(), ds);
}
ds = datasetDao.save(ds);
// ds = datasetDao.save(ds);

execution.setDataset(step.getOutputs().get(entry.getKey()), ds.getId());
saveExecution = true;
Expand Down

0 comments on commit d2c8336

Please sign in to comment.