Skip to content

Commit

Permalink
Simplify cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@julian-christl
julian-christl committed Sep 16, 2024
1 parent 8ff8197 commit 1301830
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 45 deletions.
3 changes: 2 additions & 1 deletion .idea/runConfigurations/_template__of_Gradle.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 0 additions & 20 deletions ...ain/java/de/tum/cit/aet/artemis/programming/repository/ProgrammingExerciseRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,26 +226,6 @@ default ProgrammingExercise findOneByProjectKeyOrThrow(String projectKey, boolea
""")
List<ProgrammingExercise> findAllByRecentExamEndDate(@Param("endDate1") ZonedDateTime endDate1, @Param("endDate2") ZonedDateTime endDate2);

@Query("""
SELECT DISTINCT pe
FROM ProgrammingExercise pe
LEFT JOIN FETCH pe.studentParticipations
WHERE pe.dueDate IS NOT NULL
AND :endDate1 <= pe.dueDate
AND pe.dueDate <= :endDate2
""")
List<ProgrammingExercise> findAllWithStudentParticipationByRecentDueDate(@Param("endDate1") ZonedDateTime endDate1, @Param("endDate2") ZonedDateTime endDate2);

@Query("""
SELECT DISTINCT pe
FROM ProgrammingExercise pe
LEFT JOIN FETCH pe.studentParticipations
WHERE pe.exerciseGroup IS NOT NULL
AND :endDate1 <= pe.exerciseGroup.exam.endDate
AND pe.exerciseGroup.exam.endDate <= :endDate2
""")
List<ProgrammingExercise> findAllWithStudentParticipationByRecentExamEndDate(@Param("endDate1") ZonedDateTime endDate1, @Param("endDate2") ZonedDateTime endDate2);

@Query("""
SELECT DISTINCT pe
FROM ProgrammingExercise pe
Expand Down
15 changes: 15 additions & 0 deletions ...aet/artemis/programming/repository/ProgrammingExerciseStudentParticipationRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import java.util.Optional;

import org.springframework.context.annotation.Profile;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.EntityGraph;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
Expand Down Expand Up @@ -164,6 +166,19 @@ List<ProgrammingExerciseStudentParticipation> findWithSubmissionsByExerciseIdAnd
Optional<ProgrammingExerciseStudentParticipation> findWithSubmissionsByExerciseIdAndStudentLoginAndTestRun(@Param("exerciseId") long exerciseId,
@Param("username") String username, @Param("testRun") boolean testRun);

@Query("""
SELECT participation.repositoryUri
FROM ProgrammingExerciseStudentParticipation participation
JOIN TREAT (participation.exercise AS ProgrammingExercise) pe
WHERE participation.repositoryUri IS NOT NULL
AND (
(pe.dueDate IS NOT NULL AND :latestDate <= pe.dueDate AND pe.dueDate <= :earliestDate)
OR (pe.exerciseGroup IS NOT NULL AND :latestDate <= pe.exerciseGroup.exam.endDate AND pe.exerciseGroup.exam.endDate <= :earliestDate)
)
""")
Page<String> findRepositoryUrisForGitCleanupByRecentDueDateOrRecentExamEndDate(@Param("earliestDate") ZonedDateTime earliestDate, @Param("latestDate") ZonedDateTime latestDate,
Pageable pageable);

@Query("""
SELECT participation
FROM ProgrammingExerciseStudentParticipation participation
Expand Down
74 changes: 50 additions & 24 deletions ...e/tum/cit/aet/artemis/programming/service/AutomaticProgrammingExerciseCleanupService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_SCHEDULING;
import static java.time.ZonedDateTime.now;

import java.net.URISyntaxException;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
Expand All @@ -16,6 +17,8 @@
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;

Expand All @@ -25,6 +28,7 @@
import de.tum.cit.aet.artemis.exercise.service.ParticipationService;
import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise;
import de.tum.cit.aet.artemis.programming.domain.ProgrammingExerciseStudentParticipation;
import de.tum.cit.aet.artemis.programming.domain.VcsRepositoryUri;
import de.tum.cit.aet.artemis.programming.repository.ProgrammingExerciseRepository;
import de.tum.cit.aet.artemis.programming.repository.ProgrammingExerciseStudentParticipationRepository;

Expand All @@ -44,6 +48,8 @@ public class AutomaticProgrammingExerciseCleanupService {

private final GitService gitService;

private static final int STUDENT_PARTICIPATION_CLEANUP_BATCH_SIZE = 500;

@Value("${artemis.external-system-request.batch-size}")
private int externalSystemRequestBatchSize;

Expand Down Expand Up @@ -96,35 +102,55 @@ public void cleanupGitRepositoriesOnArtemisServer() {
SecurityUtils.setAuthorizationObject();
log.info("Cleanup git repositories on Artemis server");
// we are specifically interested in exercises older than 8 weeks
var endDate2 = ZonedDateTime.now().minusWeeks(8).truncatedTo(ChronoUnit.DAYS);
var earliestDate = ZonedDateTime.now().minusWeeks(8).truncatedTo(ChronoUnit.DAYS);
// NOTE: for now we would like to cover more cases to also cleanup older repositories
var endDate1 = endDate2.minusYears(1).truncatedTo(ChronoUnit.DAYS);

// Cleanup all student repos in the REPOS folder (based on the student participations) 8 weeks after the exercise due date
log.info("Search for exercises with due date from {} until {}", endDate1, endDate2);
var programmingExercises = programmingExerciseRepository.findAllWithStudentParticipationByRecentDueDate(endDate1, endDate2);
programmingExercises.addAll(programmingExerciseRepository.findAllWithStudentParticipationByRecentExamEndDate(endDate1, endDate2));
log.info("Found {} programming exercises {} to clean {} local student repositories", programmingExercises.size(),
programmingExercises.stream().map(ProgrammingExercise::getProjectKey).collect(Collectors.joining(", ")),
programmingExercises.stream().mapToLong(programmingExercise -> programmingExercise.getStudentParticipations().size()).sum());
for (var programmingExercise : programmingExercises) {
for (var studentParticipation : programmingExercise.getStudentParticipations()) {
var programmingExerciseParticipation = (ProgrammingExerciseStudentParticipation) studentParticipation;
gitService.deleteLocalRepository(programmingExerciseParticipation.getVcsRepositoryUri());
}
}
var latestDate = earliestDate.minusYears(1).truncatedTo(ChronoUnit.DAYS);

// Cleanup all student repos in the REPOS folder (based on the student participations) 8 weeks after the exercise due date or exam end date
cleanStudentParticipationsRepositories(earliestDate, latestDate);

// Cleanup template, tests and solution repos in the REPOS folder 8 weeks after the course or exam is over
log.info("Search for exercises with course or exam date from {} until {}", endDate1, endDate2);
programmingExercises = programmingExerciseRepository.findAllByRecentCourseEndDate(endDate1, endDate2);
programmingExercises.addAll(programmingExerciseRepository.findAllByRecentExamEndDate(endDate1, endDate2));
log.info("Search for exercises with course or exam date from {} until {}", latestDate, earliestDate);
var programmingExercises = programmingExerciseRepository.findAllByRecentCourseEndDate(latestDate, earliestDate);
programmingExercises.addAll(programmingExerciseRepository.findAllByRecentExamEndDate(latestDate, earliestDate));
log.info("Found {} programming exercise to clean local template, test and solution: {}", programmingExercises.size(),
programmingExercises.stream().map(ProgrammingExercise::getProjectKey).collect(Collectors.joining(", ")));
for (var programmingExercise : programmingExercises) {
gitService.deleteLocalRepository(programmingExercise.getVcsTemplateRepositoryUri());
gitService.deleteLocalRepository(programmingExercise.getVcsSolutionRepositoryUri());
gitService.deleteLocalRepository(programmingExercise.getVcsTestRepositoryUri());
gitService.deleteLocalProgrammingExerciseReposFolder(programmingExercise);
if (!programmingExercises.isEmpty()) {
for (var programmingExercise : programmingExercises) {
gitService.deleteLocalRepository(programmingExercise.getVcsTemplateRepositoryUri());
gitService.deleteLocalRepository(programmingExercise.getVcsSolutionRepositoryUri());
gitService.deleteLocalRepository(programmingExercise.getVcsTestRepositoryUri());
gitService.deleteLocalProgrammingExerciseReposFolder(programmingExercise);
}
log.info("Finished cleaning local template, test and solution repositories");
}
}

private void cleanStudentParticipationsRepositories(ZonedDateTime earliestDate, ZonedDateTime latestDate) {
log.info("Search for exercises with due date from {} until {}", latestDate, earliestDate);
// Get all relevant participation ids
Pageable pageable = Pageable.ofSize(STUDENT_PARTICIPATION_CLEANUP_BATCH_SIZE);
Page<String> uriBatch = programmingExerciseStudentParticipationRepository.findRepositoryUrisForGitCleanupByRecentDueDateOrRecentExamEndDate(earliestDate, latestDate,
pageable);
log.info("Found {} student participations to clean local student repositories in {} batches.", uriBatch.getTotalElements(), uriBatch.getTotalPages());
if (uriBatch.getTotalElements() > 0) {
do {
uriBatch.forEach(this::deleteLocalRepositoryByUriString);
uriBatch = programmingExerciseStudentParticipationRepository.findRepositoryUrisForGitCleanupByRecentDueDateOrRecentExamEndDate(earliestDate, latestDate,
uriBatch.nextPageable());
}
while (uriBatch.hasNext());
log.info("Finished cleaning local student repositories");
}
}

private void deleteLocalRepositoryByUriString(String uri) {
try {
VcsRepositoryUri vcsRepositoryUrl = new VcsRepositoryUri(uri);
gitService.deleteLocalRepository(vcsRepositoryUrl);
}
catch (URISyntaxException e) {
log.error("Cannot create URI for repositoryUri: {} due to the following error: {}", uri, e.getMessage());
}
}

Expand Down

0 comments on commit 1301830

Please sign in to comment.