Programming exercises: Add visualization of test case errors

src/main/java/de/tum/in/www1/artemis/web/rest/ResultResource.java

@@ -5,6 +5,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -45,12 +46,15 @@
 import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastInstructor;
 import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastStudent;
 import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastTutor;
+import de.tum.in.www1.artemis.security.annotations.enforceRoleInExercise.EnforceAtLeastEditorInExercise;
 import de.tum.in.www1.artemis.service.AuthorizationCheckService;
 import de.tum.in.www1.artemis.service.ParticipationAuthorizationCheckService;
 import de.tum.in.www1.artemis.service.ParticipationService;
 import de.tum.in.www1.artemis.service.ResultService;
 import de.tum.in.www1.artemis.service.exam.ExamDateService;
 import de.tum.in.www1.artemis.web.rest.dto.ResultWithPointsPerGradingCriterionDTO;
+import de.tum.in.www1.artemis.web.rest.dto.feedback.FeedbackDetailDTO;
+import de.tum.in.www1.artemis.web.rest.dto.feedback.FeedbackDetailsWithResultIdsDTO;
 import de.tum.in.www1.artemis.web.rest.errors.BadRequestAlertException;
 import de.tum.in.www1.artemis.web.rest.util.HeaderUtil;
 
@@ -276,4 +280,39 @@ public ResponseEntity<Result> createResultForExternalSubmission(@PathVariable Lo
         return ResponseEntity.created(new URI("/api/results/" + savedResult.getId()))
                 .headers(HeaderUtil.createEntityCreationAlert(applicationName, true, ENTITY_NAME, savedResult.getId().toString())).body(savedResult);
     }
+
+    /**
+     * GET exercises/:exerciseId/feedback-details : Retrieves all negative feedback details and the latest result IDs for a given exercise.
+     *
+     * @param exerciseId The ID of the exercise for which feedback details and result IDs should be retrieved.
+     * @return A ResponseEntity containing a list of all feedback details and the corresponding result IDs for the exercise.
+     */
+    @GetMapping("exercises/{exerciseId}/feedback-details")
+    @EnforceAtLeastEditorInExercise
+    public ResponseEntity<FeedbackDetailsWithResultIdsDTO> getAllFeedbackDetailsForExercise(@PathVariable Long exerciseId) {
+        log.debug("REST request to get all Feedback details for Exercise {}", exerciseId);
+
+        List<StudentParticipation> participations = studentParticipationRepository
+                .findByExerciseIdWithLatestAutomaticResultAndFeedbacksAndTestCasesWithoutIndividualDueDate(exerciseId);
+        removeSubmissionAndExerciseData(participations);
+
+        List<Long> resultIds = new ArrayList<>();
+
+        List<FeedbackDetailDTO> allFeedbackDetails = new ArrayList<>(participations.stream().filter(participation -> !participation.isPracticeMode())
+                .flatMap(participation -> participation.getResults().stream()).peek(result -> resultIds.add(result.getId())).flatMap(result -> result.getFeedbacks().stream())
+                .filter(feedback -> Boolean.FALSE.equals(feedback.isPositive()))
+                .map(feedback -> new FeedbackDetailDTO(feedback.getDetailText(), (feedback.getTestCase() != null ? feedback.getTestCase().getTestName() : null))).toList());
+
+        FeedbackDetailsWithResultIdsDTO response = new FeedbackDetailsWithResultIdsDTO(allFeedbackDetails, resultIds);
+        return ResponseEntity.ok(response);
+    }
+
+    private void removeSubmissionAndExerciseData(List<StudentParticipation> participations) {
+        // remove unnecessary data to reduce response size
+        participations.forEach(participation -> {
+            participation.setSubmissions(null);
+            participation.setExercise(null);
+        });
+    }
+
 }

src/main/java/de/tum/in/www1/artemis/web/rest/dto/feedback/FeedbackDetailDTO.java

@@ -0,0 +1,7 @@
+package de.tum.in.www1.artemis.web.rest.dto.feedback;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+public record FeedbackDetailDTO(String detailText, String testCaseName) {
+}

src/main/java/de/tum/in/www1/artemis/web/rest/dto/feedback/FeedbackDetailsWithResultIdsDTO.java

@@ -0,0 +1,9 @@
+package de.tum.in.www1.artemis.web.rest.dto.feedback;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+public record FeedbackDetailsWithResultIdsDTO(List<FeedbackDetailDTO> feedbackDetails, List<Long> resultIds) {
+}

src/main/webapp/app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.component.html

@@ -0,0 +1,27 @@
+<div class="m-3">
+    <h2 class="mb-3" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.title" [translateValues]="{ exerciseTitle: exerciseTitle }"></h2>
+    <table class="table table-striped mb-3">
+        <thead>
+            <tr>
+                <th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.occurrence"></th>
+                <th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.feedback"></th>
+                <th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.task"></th>
+                <th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.testcase"></th>
+                <th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.errorCategory"></th>
+            </tr>
+        </thead>
+        <tbody class="table-group-divider">
+            @for (item of feedbackDetails; track item) {
+                <tr>
+                    <td class="text-center">{{ item.count }} ({{ item.relativeCount | number: '1.0-0' }}%)</td>
+                    <td>{{ item.detailText }}</td>
+                    <td class="text-center">{{ item.task }}</td>
+                    <td>{{ item.testCaseName }}</td>
+                    <td>Student Error</td>
+                    <!-- This is a placeholder, will be covered in follow up PRs -->
+                </tr>
+            }
+        </tbody>
+    </table>
+    <div jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.totalItems" [translateValues]="{ count: feedbackDetails.length }"></div>
+</div>

src/main/webapp/app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.component.ts

@@ -0,0 +1,99 @@
+import { Component, Input, OnInit } from '@angular/core';
+import { HttpResponse } from '@angular/common/http';
+import { Observable } from 'rxjs';
+import { concatMap, tap } from 'rxjs/operators';
+
+import { ArtemisSharedModule } from 'app/shared/shared.module';
+import { FeedbackAnalysisService, FeedbackDetailsWithResultIdsDTO, SimplifiedTask } from 'app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.service';
+import { AlertService } from 'app/core/util/alert.service';
+
+export interface FeedbackDetail {
+    count: number;
+    relativeCount: number;
+    detailText: string;
+    testCaseName: string;
+    task: number;
+}
+
+@Component({
+    selector: 'jhi-feedback-analysis',
+    templateUrl: './feedback-analysis.component.html',
+    standalone: true,
+    imports: [ArtemisSharedModule],
+    providers: [FeedbackAnalysisService],
+})
+export class FeedbackAnalysisComponent implements OnInit {
+    @Input() exerciseTitle?: string;
+    @Input() exerciseId?: number;
+    resultIds: number[] = [];
+    tasks: SimplifiedTask[] = [];
+    feedbackDetails: FeedbackDetail[] = [];
+
+    constructor(
+        private simplifiedProgrammingExerciseTaskService: FeedbackAnalysisService,
+        private alertService: AlertService,
+    ) {}
+
+    ngOnInit(): void {
+        if (this.exerciseId) {
+            this.loadTasks(this.exerciseId)
+                .pipe(concatMap(() => this.loadFeedbackDetails(this.exerciseId!)))
+                .subscribe();
+        }
+    }
+
+    loadTasks(exerciseId: number): Observable<SimplifiedTask[]> {
+        return this.simplifiedProgrammingExerciseTaskService.getSimplifiedTasks(exerciseId).pipe(
+            tap((tasks) => {
+                this.tasks = tasks;
+            }),
+        );
+    }
+
+    loadFeedbackDetails(exerciseId: number): Observable<HttpResponse<FeedbackDetailsWithResultIdsDTO>> {
+        return this.simplifiedProgrammingExerciseTaskService.getFeedbackDetailsForExercise(exerciseId).pipe(
+            tap((response) => {
+                this.resultIds = response.body?.resultIds || [];
+                const feedbackDetails = response.body?.feedbackDetails || [];
+                this.saveFeedback(feedbackDetails);
+            }),
+        );
+    }
+
+    saveFeedback(feedbackDetails: FeedbackDetail[]): void {
+        const feedbackMap: Map<string, FeedbackDetail> = new Map();
+
+        feedbackDetails.forEach((feedback) => {
+            const feedbackText = feedback.detailText ?? '';
+            const testCaseName = feedback.testCaseName ?? '';
+            const key = `${feedbackText}_${testCaseName}`;
+
+            const existingFeedback = feedbackMap.get(key);
+            if (existingFeedback) {
+                existingFeedback.count += 1;
+                existingFeedback.relativeCount = this.getRelativeCount(existingFeedback.count);
+            } else {
+                const task = this.taskIndex(testCaseName);
+                feedbackMap.set(key, {
+                    count: 1,
+                    relativeCount: this.getRelativeCount(1),
+                    detailText: feedbackText,
+                    testCaseName: testCaseName,
+                    task: task,
+                });
+            }
+        });
+        this.feedbackDetails = Array.from(feedbackMap.values()).sort((a, b) => b.count - a.count);
+    }
+
+    taskIndex(testCaseName: string): number {
+        if (!testCaseName) {
+            return 0;
+        }
+        return this.tasks.findIndex((tasks) => tasks.testCases?.some((testCase) => testCase.testName === testCaseName)) + 1;
+    }
+
+    getRelativeCount(count: number): number {
+        return (count / this.resultIds.length) * 100;
+    }
+}

src/main/webapp/app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.service.ts

@@ -0,0 +1,39 @@
+import { Injectable } from '@angular/core';
+import { HttpClient, HttpResponse } from '@angular/common/http';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { ProgrammingExerciseServerSideTask } from 'app/entities/hestia/programming-exercise-task.model';
+import { FeedbackDetail } from 'app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.component';
+
+export interface SimplifiedTask {
+    taskName: string;
+    testCases: ProgrammingExerciseServerSideTask['testCases'];
+}
+
+export interface FeedbackDetailsWithResultIdsDTO {
+    feedbackDetails: FeedbackDetail[];
+    resultIds: number[];
+}
+
+@Injectable()
+export class FeedbackAnalysisService {
+    private readonly resourceUrl = 'api/programming-exercises';
+    private readonly exerciseResourceUrl = 'api/exercises';
+
+    constructor(private http: HttpClient) {}
+
+    getFeedbackDetailsForExercise(exerciseId: number): Observable<HttpResponse<FeedbackDetailsWithResultIdsDTO>> {
+        return this.http.get<FeedbackDetailsWithResultIdsDTO>(`${this.exerciseResourceUrl}/${exerciseId}/feedback-details`, { observe: 'response' });
+    }
+
+    public getSimplifiedTasks(exerciseId: number): Observable<SimplifiedTask[]> {
+        return this.http.get<ProgrammingExerciseServerSideTask[]>(`${this.resourceUrl}/${exerciseId}/tasks-with-unassigned-test-cases`).pipe(
+            map((tasks) =>
+                tasks.map((task) => ({
+                    taskName: task.taskName ?? '',
+                    testCases: task.testCases ?? [],
+                })),
+            ),
+        );
+    }
+}

src/main/webapp/app/exercises/programming/manage/grading/programming-exercise-configure-grading.component.html

@@ -6,30 +6,35 @@ <h3 class="fw-medium" jhiTranslate="artemisApp.programmingExercise.configureGrad
         <div class="top-bar">
             <div class="d-flex align-items-center">
                 <div class="tab-item test-cases" (click)="selectTab('test-cases')" [ngClass]="activeTab === 'test-cases' ? 'active' : ''">
-                    <b>Test Cases</b>
+                    <b jhiTranslate="artemisApp.programmingExercise.configureGrading.testCases.title"></b>
                 </div>
                 @if (programmingExercise.staticCodeAnalysisEnabled) {
                     <div class="tab-item code-analysis" (click)="selectTab('code-analysis')" [ngClass]="activeTab === 'code-analysis' ? 'active' : ''">
-                        <b>Code Analysis</b>
+                        <b jhiTranslate="artemisApp.programmingExercise.configureGrading.categories.titleHeader"></b>
                     </div>
                 }
                 <div class="tab-item submission-policy" (click)="selectTab('submission-policy')" [ngClass]="activeTab === 'submission-policy' ? 'active' : ''">
-                    <b>Submission Policy</b>
+                    <b jhiTranslate="artemisApp.programmingExercise.submissionPolicy.title"></b>
                 </div>
+                @if (programmingExercise.isAtLeastEditor) {
+                    <div class="tab-item" (click)="selectTab('feedback-analysis')" [ngClass]="activeTab === 'feedback-analysis' ? 'active' : ''">
+                        <b jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.titleHeader"></b>
+                    </div>
+                }
             </div>
             <ng-template>
                 <div></div>
             </ng-template>
             <div class="d-flex align-items-center">
-                @if (activeTab !== 'submission-policy') {
+                @if (activeTab === 'test-cases' || activeTab === 'code-analysis') {
                     <jhi-programming-exercise-configure-grading-status
                         [exerciseIsReleasedAndHasResults]="isReleasedAndHasResults"
                         [hasUnsavedTestCaseChanges]="hasUnsavedChanges"
                         [hasUnsavedCategoryChanges]="!!changedCategoryIds.length"
                         [hasUpdatedGradingConfig]="hasUpdatedGradingConfig"
                     />
                 }
-                @if (programmingExercise.isAtLeastInstructor) {
+                @if (programmingExercise.isAtLeastInstructor && activeTab !== 'feedback-analysis') {
                     <jhi-programming-exercise-configure-grading-actions
                         [exercise]="programmingExercise"
                         [hasUpdatedGradingConfig]="hasUpdatedGradingConfig"
@@ -256,5 +261,10 @@ <h2 class="mb-5 fw-medium">
                 </div>
             }
         </div>
+        <div class="grading-body-container mt-3">
+            @if (programmingExercise.isAtLeastEditor && activeTab === 'feedback-analysis') {
+                <jhi-feedback-analysis [exerciseTitle]="programmingExercise.title" [exerciseId]="programmingExercise.id"></jhi-feedback-analysis>
+            }
+        </div>
     }
 </div>

src/main/webapp/app/exercises/programming/manage/grading/programming-exercise-configure-grading.component.ts

@@ -56,8 +56,7 @@ const DefaultFieldValues: { [key: string]: number } = {
     [EditableField.MAX_PENALTY]: 0,
 };
 
-export type GradingTab = 'test-cases' | 'code-analysis' | 'submission-policy';
-
+export type GradingTab = 'test-cases' | 'code-analysis' | 'submission-policy' | 'feedback-analysis';
 export type Table = 'testCases' | 'codeAnalysis';
 
 @Component({
@@ -232,7 +231,8 @@ export class ProgrammingExerciseConfigureGradingComponent implements OnInit, OnD
                 this.isLoading = false;
             }
 
-            if (params['tab'] === 'test-cases' || params['tab'] === 'code-analysis' || params['tab'] === 'submission-policy') {
+            const gradingTabs: GradingTab[] = ['test-cases', 'code-analysis', 'submission-policy', 'feedback-analysis'];
+            if (gradingTabs.includes(params['tab'])) {
                 this.selectTab(params['tab']);
             } else {
                 this.selectTab('test-cases');

src/main/webapp/app/exercises/programming/manage/grading/programming-exercise-grading.module.ts

@@ -19,6 +19,7 @@ import { SubmissionPolicyUpdateModule } from 'app/exercises/shared/submission-po
 import { ProgrammingExerciseGradingTasksTableComponent } from 'app/exercises/programming/manage/grading/tasks/programming-exercise-grading-tasks-table.component';
 import { BarChartModule } from '@swimlane/ngx-charts';
 import { ProgrammingExerciseTaskComponent } from 'app/exercises/programming/manage/grading/tasks/programming-exercise-task/programming-exercise-task.component';
+import { FeedbackAnalysisComponent } from 'app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.component';
 
 @NgModule({
     imports: [
@@ -33,6 +34,7 @@ import { ProgrammingExerciseTaskComponent } from 'app/exercises/programming/mana
         ArtemisProgrammingExerciseActionsModule,
         SubmissionPolicyUpdateModule,
         BarChartModule,
+        FeedbackAnalysisComponent,
     ],
     declarations: [
         ProgrammingExerciseConfigureGradingComponent,

src/main/webapp/i18n/de/programmingExercise.json

@@ -260,7 +260,8 @@
                     "settingNegative": "Der Testfall {{testCase}} darf keine Einstellungen mit negativen Werten haben."
                 },
                 "categories": {
-                    "title": "Code-Analyse-Kategorien",
+                    "titleHeader": "Quelltext-Analyse",
+                    "title": "Quelltext-Analyse-Kategorien",
                     "notGraded": "Nicht bewertet.",
                     "noFeedback": "Ohne sichtbares Feedback.",
                     "updated": "Die Kategorien wurden erfolgreich gespeichert.",
@@ -306,6 +307,16 @@
                     "testType": "Type",
                     "passedPercent": "Bestanden %"
                 },
+                "feedbackAnalysis": {
+                    "titleHeader": "Feedback Analyse",
+                    "title": "Fehleranalyse für {{exerciseTitle}}",
+                    "occurrence": "Häufigkeit",
+                    "feedback": "Feedback",
+                    "task": "Aufgabe",
+                    "testcase": "Testfall",
+                    "errorCategory": "Fehlerkategorie",
+                    "totalItems": "Insgesamt {{count}} Elemente"
+                },
                 "help": {
                     "name": "Aufgabennamen werden fett geschrieben, während Testnamen normal sind. Ob es ein Aufgabenname oder Testname ist hängt davon ab, ob die Reihe eine Aufgabe oder einen Test darstellt.",
                     "state": "Gibt an, ob Issues in dieser Kategorie den Studierenden angezeigt und bewertet werden sollen.",

src/main/webapp/i18n/en/programmingExercise.json

@@ -262,6 +262,7 @@
                     "settingNegative": "Test case {{testCase}} must not have settings set to negative values."
                 },
                 "categories": {
+                    "titleHeader": "Code Analysis",
                     "title": "Code Analysis Categories",
                     "notGraded": "Not graded.",
                     "noFeedback": "No visible feedback.",
@@ -308,6 +309,16 @@
                     "testType": "Type",
                     "passedPercent": "Passed %"
                 },
+                "feedbackAnalysis": {
+                    "titleHeader": "Feedback Analysis",
+                    "title": "Feedback Analysis for {{exerciseTitle}}",
+                    "occurrence": "Occurrence",
+                    "feedback": "Feedback",
+                    "task": "Task",
+                    "testcase": "Test Case",
+                    "errorCategory": "Error Category",
+                    "totalItems": "In total {{count}} items"
+                },
                 "help": {
                     "name": "Task names are written in bold whereas Test names are normal. Task or test name depending on whether the row is a task or test.",
                     "state": "Determines whether issues in this category should be shown to the students and used for grading.",