diff --git a/build.gradle b/build.gradle index e53437485601..4ed65552ce49 100644 --- a/build.gradle +++ b/build.gradle @@ -78,7 +78,8 @@ spotless { "**/src/main/resources/templates/**", "/docker/**", "checked-out-repos/**", - "**/src/main/java/org/eclipse/**" + "**/src/main/java/org/eclipse/**", + "supporting_scripts/**" ) } } diff --git a/supporting_scripts/course-scripts/quick-course-setup/README.md b/supporting_scripts/course-scripts/quick-course-setup/README.md index e1622e686272..4b6e94aa7b8c 100644 --- a/supporting_scripts/course-scripts/quick-course-setup/README.md +++ b/supporting_scripts/course-scripts/quick-course-setup/README.md @@ -166,6 +166,30 @@ The script will automatically perform all the necessary steps: 5. Create a programming exercise or use an existing one. 6. Add participation and commit for each student. +### Optional: Generating Different Results For All Created Students (Should only be done Locally!!) + +If you want to generate different results for all the students created by the script: + +1. Run the following Script which will navigate to the [testFiles](../../../src/main/resources/templates/java/test/testFiles) folder and copy the [RandomizedTestCases](./testFiles-template/randomized/RandomizedTestCases.java) file into it. + It will delete the existing folders (behavior and structural) from the programming exercise’s test case template. The new test cases will randomly pass or fail, causing different results for each student. +```shell +python3 randomize_results_before.py +``` +2. Rebuild Artemis to apply the changes. +3. Run the main method in large_course_main.py. Now, all created students should have varying results in the programming exercise. +```shell +python3 large_course_main.py +``` +4. Make sure to revert these changes after running the script. The following script copies the original test case files from the [default](./testFiles-template/default) folder back into the [testFiles](../../../src/main/resources/templates/java/test/testFiles) folder and deletes the [RandomizedTestCases](./testFiles-template/randomized/RandomizedTestCases.java) file that was copied to [testFiles](../../../src/main/resources/templates/java/test/testFiles) in Step 1. + If you don't run this script after running the script in Step 1, you risk breaking the real template of the programming exercise if these changes are pushed and merged. +```shell +python3 randomize_results_after.py +``` + +### Optional: Using an Existing Programming Exercise (Can also be done on Test Server) +Alternatively, you can use an existing programming exercise and push the [RandomizedTestCases](./testFiles-template/randomized/RandomizedTestCases.java) file to the test repository of the programming exercise. +Make sure to adjust the [config.ini](./config.ini) file to use the existing programming exercise with the corresponding exercise ID, allowing the script to push with the created students to this existing programming exercise. + ### Optional: Deleting All Created Students If you want to delete all the students created by the script: diff --git a/supporting_scripts/course-scripts/quick-course-setup/create_course.py b/supporting_scripts/course-scripts/quick-course-setup/create_course.py index 7ec0f270e5ad..53095f2e3f37 100644 --- a/supporting_scripts/course-scripts/quick-course-setup/create_course.py +++ b/supporting_scripts/course-scripts/quick-course-setup/create_course.py @@ -8,6 +8,7 @@ from requests import Session from utils import login_as_admin from add_users_to_course import add_users_to_groups_of_course +from randomize_results_after import run_cleanup # Load configuration config = configparser.ConfigParser() @@ -89,6 +90,7 @@ def create_course(session: Session) -> requests.Response: logging.info(f"Created course {COURSE_NAME} with shortName {course_short_name} \n {response.json()}") elif response.status_code == 400: logging.info(f"Course with shortName {course_short_name} already exists. Please provide the course ID in the config file and set create_course to FALSE if you intend to add programming exercises to this course.") + run_cleanup() sys.exit(0) else: logging.error("Problem with the group 'students' and interacting with a test server? " diff --git a/supporting_scripts/course-scripts/quick-course-setup/large_course_main.py b/supporting_scripts/course-scripts/quick-course-setup/large_course_main.py index 37f419b95025..e53f6e7e6a30 100644 --- a/supporting_scripts/course-scripts/quick-course-setup/large_course_main.py +++ b/supporting_scripts/course-scripts/quick-course-setup/large_course_main.py @@ -7,6 +7,7 @@ from create_users import create_students, user_credentials from add_users_to_course import add_students_to_groups_of_course from manage_programming_exercise import create_programming_exercise, add_participation, commit, exercise_Ids +from randomize_results_after import run_cleanup # Load configuration and constants config = configparser.ConfigParser() @@ -68,5 +69,8 @@ def main() -> None: commit(user_session, participation_id, CLIENT_URL, COMMITS_PER_STUDENT) logging.info(f"Added commit for {username} in the programming exercise {exercise_Id} successfully") + # This is a measure in case developers forget to revert changes to programming exercise template + run_cleanup() + if __name__ == "__main__": main() diff --git a/supporting_scripts/course-scripts/quick-course-setup/manage_programming_exercise.py b/supporting_scripts/course-scripts/quick-course-setup/manage_programming_exercise.py index 49b722e8a4d0..ab2dfc469d5d 100644 --- a/supporting_scripts/course-scripts/quick-course-setup/manage_programming_exercise.py +++ b/supporting_scripts/course-scripts/quick-course-setup/manage_programming_exercise.py @@ -2,6 +2,7 @@ from logging_config import logging from typing import Dict, Any from requests import Session +from randomize_results_after import run_cleanup exercise_Ids: list[int] = [] @@ -39,6 +40,7 @@ def create_programming_exercise(session: Session, course_id: int, server_url: st exercise_Ids.append(response.json().get('id')) elif response.status_code == 400: logging.info(f"Programming exercise with shortName {default_programming_exercise['shortName']} already exists. Please provide the exercise IDs in the config file and set create_exercises to FALSE.") + run_cleanup() sys.exit(0) else: raise Exception(f"Could not create programming exercise; Status code: {response.status_code}\nResponse content: {response.text}") diff --git a/supporting_scripts/course-scripts/quick-course-setup/randomize_results_after.py b/supporting_scripts/course-scripts/quick-course-setup/randomize_results_after.py new file mode 100644 index 000000000000..9efd1fe2fba0 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/randomize_results_after.py @@ -0,0 +1,31 @@ +import os +import shutil +from logging_config import logging + +test_files_folder: str = "../../../src/main/resources/templates/java/test/testFiles" +default_folder: str = "./testFiles-template/default" +random_test_case_dest: str = os.path.join(test_files_folder, "RandomizedTestCases.java") + +def delete_random_test_case() -> None: + if os.path.exists(random_test_case_dest): + os.remove(random_test_case_dest) + logging.info(f"Deleted file: {random_test_case_dest}") + else: + logging.info(f"RandomizedTestCases.java file not found at {random_test_case_dest}") + +def copy_default_folders() -> None: + for folder_name in os.listdir(default_folder): + src_folder_path: str = os.path.join(default_folder, folder_name) + dest_folder_path: str = os.path.join(test_files_folder, folder_name) + if os.path.isdir(src_folder_path): + shutil.copytree(src_folder_path, dest_folder_path) + logging.info(f"Copied folder {src_folder_path} to {dest_folder_path}") + +def run_cleanup() -> None: + delete_random_test_case() + copy_default_folders() + +if __name__ == "__main__": + # Run this after running the script + delete_random_test_case() + copy_default_folders() diff --git a/supporting_scripts/course-scripts/quick-course-setup/randomize_results_before.py b/supporting_scripts/course-scripts/quick-course-setup/randomize_results_before.py new file mode 100644 index 000000000000..c969a1ab9059 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/randomize_results_before.py @@ -0,0 +1,29 @@ +import os +import shutil +from logging_config import logging + +test_files_folder: str = "../../../src/main/resources/templates/java/test/testFiles" +random_test_case_file: str = "./testFiles-template/randomized/RandomizedTestCases.java" +random_test_case_dest: str = os.path.join(test_files_folder, "RandomizedTestCases.java") + +def delete_existing_folders() -> None: + folders_to_delete: list[str] = ["behavior", "structural"] + for folder in folders_to_delete: + folder_path: str = os.path.join(test_files_folder, folder) + if os.path.exists(folder_path) and os.path.isdir(folder_path): + shutil.rmtree(folder_path) + logging.info(f"Deleted folder: {folder_path}") + else: + logging.info(f"Folder not found: {folder_path}") + +def copy_random_test_case() -> None: + if os.path.exists(random_test_case_file): + shutil.copy(random_test_case_file, random_test_case_dest) + logging.info(f"Copied {random_test_case_file} to {random_test_case_dest}") + else: + logging.info(f"RandomizedTestCases.java file not found at {random_test_case_file}") + +if __name__ == "__main__": + # Run this before running the script + delete_existing_folders() + copy_random_test_case() diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/behavior/SortingExampleBehaviorTest.java b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/behavior/SortingExampleBehaviorTest.java new file mode 100644 index 000000000000..12c36a1203e0 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/behavior/SortingExampleBehaviorTest.java @@ -0,0 +1,98 @@ +package ${packageName}; + +import org.junit.jupiter.api.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.text.*; +import java.util.*; + +import static de.tum.in.test.api.util.ReflectionTestUtils.*; + +import de.tum.in.test.api.BlacklistPath; +import de.tum.in.test.api.PathType; +import de.tum.in.test.api.StrictTimeout; +import de.tum.in.test.api.WhitelistPath; +import de.tum.in.test.api.jupiter.Public; + +/** + * @author Stephan Krusche (krusche@in.tum.de) + * @version 5.1 (11.06.2021) + */ +@Public +@WhitelistPath("target") // mainly for Artemis +@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources +class SortingExampleBehaviorTest { + + private List dates; + private List datesWithCorrectOrder; + + @BeforeEach + void setup() throws ParseException { + SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy"); + Date date1 = dateFormat.parse("08.11.2018"); + Date date2 = dateFormat.parse("15.04.2017"); + Date date3 = dateFormat.parse("15.02.2016"); + Date date4 = dateFormat.parse("15.09.2017"); + + this.dates = Arrays.asList(date1, date2, date3, date4); + this.datesWithCorrectOrder = Arrays.asList(date3, date2, date4, date1); + } + + @Test + @StrictTimeout(1) + void testBubbleSort() { + BubbleSort bubbleSort = new BubbleSort(); + bubbleSort.performSort(dates); + if (!datesWithCorrectOrder.equals(dates)) { + fail("BubbleSort does not sort correctly"); + } + } + + @Test + @StrictTimeout(1) + void testMergeSort() { + MergeSort mergeSort = new MergeSort(); + mergeSort.performSort(dates); + if (!datesWithCorrectOrder.equals(dates)) { + fail("MergeSort does not sort correctly"); + } + } + + @Test + @StrictTimeout(1) + void testUseMergeSortForBigList() throws ReflectiveOperationException { + List bigList = new ArrayList(); + for (int i = 0; i < 11; i++) { + bigList.add(new Date()); + } + Object chosenSortStrategy = configurePolicyAndContext(bigList); + if (!(chosenSortStrategy instanceof MergeSort)) { + fail("The sort algorithm of Context was not MergeSort for a list with more than 10 dates."); + } + } + + @Test + @StrictTimeout(1) + void testUseBubbleSortForSmallList() throws ReflectiveOperationException { + List smallList = new ArrayList(); + for (int i = 0; i < 3; i++) { + smallList.add(new Date()); + } + Object chosenSortStrategy = configurePolicyAndContext(smallList); + if (!(chosenSortStrategy instanceof BubbleSort)) { + fail("The sort algorithm of Context was not BubbleSort for a list with less or equal than 10 dates."); + } + } + + private Object configurePolicyAndContext(List dates) throws ReflectiveOperationException { + Object context = newInstance("${packageName}.Context"); + invokeMethod(context, getMethod(context, "setDates", List.class), dates); + + Object policy = newInstance("${packageName}.Policy", context); + invokeMethod(policy, getMethod(policy, "configure")); + + Object chosenSortStrategy = invokeMethod(context, getMethod(context, "getSortAlgorithm")); + return chosenSortStrategy; + } +} diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/AttributeTest.java b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/AttributeTest.java new file mode 100644 index 000000000000..279ebfd4a091 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/AttributeTest.java @@ -0,0 +1,39 @@ +package ${packageName}; + +import java.net.URISyntaxException; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; + +import de.tum.in.test.api.BlacklistPath; +import de.tum.in.test.api.PathType; +import de.tum.in.test.api.StrictTimeout; +import de.tum.in.test.api.WhitelistPath; +import de.tum.in.test.api.jupiter.Public; +import de.tum.in.test.api.structural.AttributeTestProvider; + +/** + * @author Stephan Krusche (krusche@in.tum.de) + * @version 5.1 (11.06.2021) + *

+ * This test evaluates if the specified attributes in the structure oracle are correctly implemented with the expected type, visibility modifiers and annotations, + * based on its definition in the structure oracle (test.json). + */ +@Public +@WhitelistPath("target") // mainly for Artemis +@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources +class AttributeTest extends AttributeTestProvider { + + /** + * This method collects the classes in the structure oracle file for which attributes are specified. + * These classes are then transformed into JUnit 5 dynamic tests. + * @return A dynamic test container containing the test for each class which is then executed by JUnit. + */ + @Override + @StrictTimeout(10) + @TestFactory + protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException { + structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json")); + return super.generateTestsForAllClasses(); + } +} diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ClassTest.java b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ClassTest.java new file mode 100644 index 000000000000..bd7946740e7e --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ClassTest.java @@ -0,0 +1,39 @@ +package ${packageName}; + +import java.net.URISyntaxException; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; + +import de.tum.in.test.api.BlacklistPath; +import de.tum.in.test.api.PathType; +import de.tum.in.test.api.StrictTimeout; +import de.tum.in.test.api.WhitelistPath; +import de.tum.in.test.api.jupiter.Public; +import de.tum.in.test.api.structural.ClassTestProvider; + +/** + * @author Stephan Krusche (krusche@in.tum.de) + * @version 5.1 (11.06.2021) + *

+ * This test evaluates the hierarchy of the class, i.e. if the class is abstract or an interface or an enum and also if the class extends another superclass and if + * it implements the interfaces and annotations, based on its definition in the structure oracle (test.json). + */ +@Public +@WhitelistPath("target") // mainly for Artemis +@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources +class ClassTest extends ClassTestProvider { + + /** + * This method collects the classes in the structure oracle file for which at least one class property is specified. + * These classes are then transformed into JUnit 5 dynamic tests. + * @return A dynamic test container containing the test for each class which is then executed by JUnit. + */ + @Override + @StrictTimeout(10) + @TestFactory + protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException { + structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json")); + return super.generateTestsForAllClasses(); + } +} diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ConstructorTest.java b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ConstructorTest.java new file mode 100644 index 000000000000..5ab03c2913bd --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/ConstructorTest.java @@ -0,0 +1,39 @@ +package ${packageName}; + +import java.net.URISyntaxException; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; + +import de.tum.in.test.api.BlacklistPath; +import de.tum.in.test.api.PathType; +import de.tum.in.test.api.StrictTimeout; +import de.tum.in.test.api.WhitelistPath; +import de.tum.in.test.api.jupiter.Public; +import de.tum.in.test.api.structural.ConstructorTestProvider; + +/** + * @author Stephan Krusche (krusche@in.tum.de) + * @version 5.1 (11.06.2021) + *

+ * This test evaluates if the specified constructors in the structure oracle are correctly implemented with the expected parameter types and annotations, + * based on its definition in the structure oracle (test.json). + */ +@Public +@WhitelistPath("target") // mainly for Artemis +@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources +class ConstructorTest extends ConstructorTestProvider { + + /** + * This method collects the classes in the structure oracle file for which constructors are specified. + * These classes are then transformed into JUnit 5 dynamic tests. + * @return A dynamic test container containing the test for each class which is then executed by JUnit. + */ + @Override + @StrictTimeout(10) + @TestFactory + protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException { + structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json")); + return super.generateTestsForAllClasses(); + } +} diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/MethodTest.java b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/MethodTest.java new file mode 100644 index 000000000000..874e33d60487 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/MethodTest.java @@ -0,0 +1,39 @@ +package ${packageName}; + +import java.net.URISyntaxException; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; + +import de.tum.in.test.api.BlacklistPath; +import de.tum.in.test.api.PathType; +import de.tum.in.test.api.StrictTimeout; +import de.tum.in.test.api.WhitelistPath; +import de.tum.in.test.api.jupiter.Public; +import de.tum.in.test.api.structural.MethodTestProvider; + +/** + * @author Stephan Krusche (krusche@in.tum.de) + * @version 5.1 (11.06.2021) + *

+ * This test evaluates if the specified methods in the structure oracle are correctly implemented with the expected name, return type, parameter types, visibility modifiers + * and annotations, based on its definition in the structure oracle (test.json) + */ +@Public +@WhitelistPath("target") // mainly for Artemis +@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources +class MethodTest extends MethodTestProvider { + + /** + * This method collects the classes in the structure oracle file for which methods are specified. + * These classes are then transformed into JUnit 5 dynamic tests. + * @return A dynamic test container containing the test for each class which is then executed by JUnit. + */ + @Override + @StrictTimeout(10) + @TestFactory + protected DynamicContainer generateTestsForAllClasses() throws URISyntaxException { + structureOracleJSON = retrieveStructureOracleJSON(this.getClass().getResource("test.json")); + return super.generateTestsForAllClasses(); + } +} diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/test.json b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/test.json new file mode 100644 index 000000000000..860c268eaec7 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/default/structural/test.json @@ -0,0 +1,81 @@ +[ { + "class" : { + "name" : "MergeSort", + "package" : "${packageName}", + "interfaces" : [ "SortStrategy" ] + } +}, { + "class" : { + "name" : "Context", + "package" : "${packageName}" + }, + "methods" : [ { + "name" : "getDates", + "modifiers" : [ "public" ], + "returnType" : "List" + }, { + "name" : "setDates", + "modifiers" : [ "public" ], + "parameters" : [ "List" ], + "returnType" : "void" + }, { + "name" : "setSortAlgorithm", + "modifiers" : [ "public" ], + "parameters" : [ "SortStrategy" ], + "returnType" : "void" + }, { + "name" : "getSortAlgorithm", + "modifiers" : [ "public" ], + "returnType" : "SortStrategy" + }, { + "name" : "sort", + "modifiers" : [ "public" ], + "returnType" : "void" + } ], + "attributes" : [ { + "name" : "sortAlgorithm", + "modifiers" : [ "private" ], + "type" : "SortStrategy" + }, { + "name" : "dates", + "modifiers" : [ "private" ], + "type" : "List" + } ] +}, { + "class" : { + "name" : "Policy", + "package" : "${packageName}" + }, + "methods" : [ { + "name" : "configure", + "modifiers" : [ "public" ], + "returnType" : "void" + } ], + "attributes" : [ { + "name" : "context", + "modifiers" : [ "private", "optional: final" ], + "type" : "Context" + } ], + "constructors" : [ { + "modifiers" : [ "public" ], + "parameters" : [ "Context" ] + } ] +}, { + "class" : { + "name" : "SortStrategy", + "package" : "${packageName}", + "isInterface" : true + }, + "methods" : [ { + "name" : "performSort", + "modifiers" : [ "public", "abstract" ], + "parameters" : [ "List" ], + "returnType" : "void" + } ] +}, { + "class" : { + "name" : "BubbleSort", + "package" : "${packageName}", + "interfaces" : [ "SortStrategy" ] + } +} ] diff --git a/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/randomized/RandomizedTestCases.java b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/randomized/RandomizedTestCases.java new file mode 100644 index 000000000000..62d7a9b1a0d9 --- /dev/null +++ b/supporting_scripts/course-scripts/quick-course-setup/testFiles-template/randomized/RandomizedTestCases.java @@ -0,0 +1,90 @@ +package ${packageName}; + +import org.junit.jupiter.api.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.text.*; +import java.util.*; + +import static de.tum.in.test.api.util.ReflectionTestUtils.*; + +import de.tum.in.test.api.BlacklistPath; +import de.tum.in.test.api.PathType; +import de.tum.in.test.api.StrictTimeout; +import de.tum.in.test.api.WhitelistPath; +import de.tum.in.test.api.jupiter.Public; +@Public +@WhitelistPath("target") // mainly for Artemis +@BlacklistPath("target/test-classes") // prevent access to test-related classes and resources +class RandomizedTestCases { + + @Test + @StrictTimeout(1) + void testCase1() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase2() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase3() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase4() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase5() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase6() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase7() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase8() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase9() { + randomTest(); + } + + @Test + @StrictTimeout(1) + void testCase10() { + randomTest(); + } + + private void randomTest() { + Random random = new Random(); + boolean shouldFail = random.nextBoolean(); + + if (shouldFail) { + fail(String.format("Different error: %s, %s", System.currentTimeMillis(), random.nextInt(Integer.MAX_VALUE))); + } + } +}