Development: Add dynamic fetches to simplify complexity in repositori…

…es (#8607)

docs/dev/guidelines/database.rst

@@ -142,6 +142,70 @@ Entity relationships often depend on the existence of another entity — for exa
 Not used in Artemis yet.
 
 
+4. Dynamic Fetching
+===================
+In Artemis, we use dynamic fetching to load the relationships of an entity on demand. As we do not want to load all relationships of an entity every time we fetch it from the database, we use as described above ``FetchType.LAZY``.
+In order to load the relationships of an entity on demand, we then use one of 3 methods:
+
+1. **EntityGraph**: The ``@EntityGraph`` annotation is the simplest way to specify a graph of relationships to fetch. It should be used when a query is auto-constructed by Spring Data JPA and does not have a custom ``@Query`` annotation. Example:
+
+ .. code-block:: java
+
+    // CourseRepository.java
+    @EntityGraph(type = LOAD, attributePaths = { "exercises", "exercises.categories", "exercises.teamAssignmentConfig" })
+    Course findWithEagerExercisesById(long courseId);
+
+2. **JOIN FETCH**: The ``JOIN FETCH`` keyword is used in a custom query to specify a graph of relationships to fetch. It should be used when a query is custom and has a custom ``@Query`` annotation. Example:
+
+ .. code-block:: java
+
+    // ProgrammingExerciseRepository.java
+    @Query("""
+            SELECT pe
+            FROM ProgrammingExercise pe
+                LEFT JOIN FETCH pe.exerciseGroup eg
+                LEFT JOIN FETCH eg.exam e
+            WHERE e.endDate > :dateTime
+            """)
+    List<ProgrammingExercise> findAllWithEagerExamByExamEndDateAfterDate(@Param("dateTime") ZonedDateTime dateTime);
+
+3. **DynamicSpecificationRepository**: For repositories that use a lot of different queries with different relationships to fetch, we use the ``DynamicSpecificationRepository``. You can let a repository additionally implement this interface and then use the ``findAllWithEagerRelationships`` and then use the ``getDynamicSpecification(Collection<? extends FetchOptions> fetchOptions)`` method in combination with a custom enum implementing the ``FetchOptions`` interface to dynamically specify which relationships to fetch inside of service methods. Example:
+
+ .. code-block:: java
+
+    // ProgrammingExerciseFetchOptions.java
+    public enum ProgrammingExerciseFetchOptions implements FetchOptions {
+
+        GradingCriteria(Exercise_.GRADING_CRITERIA),
+        AuxiliaryRepositories(Exercise_.AUXILIARY_REPOSITORIES),
+        // ...
+
+        private final String fetchPath;
+
+        ProgrammingExerciseFetchOptions(String fetchPath) {
+            this.fetchPath = fetchPath;
+        }
+
+        public String getFetchPath() {
+            return fetchPath;
+        }
+    }
+
+
+    // ProgrammingExerciseRepository.java
+    @NotNull
+    default ProgrammingExercise findByIdWithDynamicFetchElseThrow(long exerciseId, Collection<ProgrammingExerciseFetchOptions> fetchOptions) throws EntityNotFoundException {
+        var specification = getDynamicSpecification(fetchOptions);
+        return findOneByIdElseThrow(specification, exerciseId, "Programming Exercise");
+    }
+
+
+    // ProgrammingExerciseService.java
+    final Set<ProgrammingExerciseFetchOptions> fetchOptions = withGradingCriteria ? Set.of(GradingCriteria, AuxiliaryRepositories) : Set.of(AuxiliaryRepositories);
+    var programmingExercise = programmingExerciseRepository.findByIdWithDynamicFetchElseThrow(exerciseId, fetchOptions);
+
+
+
 Best Practices
 ==============
 * If you want to create a ``@OneToMany`` relationship or ``@ManyToMany`` relationship, first think about if it is important for the association to be ordered. If you do not need the association to be ordered, then always go for a ``Set`` instead of ``List``. If you are unsure, start with a ``Set``. 

src/main/java/de/tum/in/www1/artemis/repository/ProgrammingExerciseRepository.java

@@ -3,30 +3,27 @@
 import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE;
 import static de.tum.in.www1.artemis.config.Constants.SHORT_NAME_PATTERN;
 import static de.tum.in.www1.artemis.config.Constants.TITLE_NAME_PATTERN;
+import static de.tum.in.www1.artemis.repository.ProgrammingExerciseRepository.ProgrammingExerciseFetchOptions;
 import static org.springframework.data.jpa.repository.EntityGraph.EntityGraphType.LOAD;
 
 import java.time.ZonedDateTime;
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Matcher;
 
 import jakarta.annotation.Nullable;
-import jakarta.persistence.criteria.JoinType;
 import jakarta.validation.constraints.NotNull;
 
 import org.hibernate.Hibernate;
 import org.springframework.context.annotation.Profile;
-import org.springframework.data.jpa.domain.Specification;
 import org.springframework.data.jpa.repository.EntityGraph;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
 
 import de.tum.in.www1.artemis.domain.Course;
-import de.tum.in.www1.artemis.domain.DomainObject_;
 import de.tum.in.www1.artemis.domain.Exercise_;
 import de.tum.in.www1.artemis.domain.ProgrammingExercise;
 import de.tum.in.www1.artemis.domain.ProgrammingExercise_;
@@ -35,6 +32,8 @@
 import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseStudentParticipation;
 import de.tum.in.www1.artemis.domain.participation.SolutionProgrammingExerciseParticipation;
 import de.tum.in.www1.artemis.domain.participation.TemplateProgrammingExerciseParticipation;
+import de.tum.in.www1.artemis.repository.base.DynamicSpecificationRepository;
+import de.tum.in.www1.artemis.repository.base.FetchOptions;
 import de.tum.in.www1.artemis.web.rest.errors.BadRequestAlertException;
 import de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException;
 
@@ -43,12 +42,7 @@
  */
 @Profile(PROFILE_CORE)
 @Repository
-public interface ProgrammingExerciseRepository extends JpaRepository<ProgrammingExercise, Long>, JpaSpecificationExecutor<ProgrammingExercise> {
-
-    default ProgrammingExercise findOneByIdElseThrow(final Specification<ProgrammingExercise> specification, long exerciseId) {
-        final Specification<ProgrammingExercise> hasIdSpec = (root, query, criteriaBuilder) -> criteriaBuilder.equal(root.get(DomainObject_.ID), exerciseId);
-        return findOne(specification.and(hasIdSpec)).orElseThrow(() -> new EntityNotFoundException("Programming Exercise", exerciseId));
-    }
+public interface ProgrammingExerciseRepository extends DynamicSpecificationRepository<ProgrammingExercise, Long, ProgrammingExerciseFetchOptions> {
 
     /**
      * Does a max join on the result table for each participation by result id (the newer the result id, the newer the result).
@@ -639,30 +633,30 @@ default ProgrammingExercise findByIdWithTemplateAndSolutionParticipationTeamAssi
     }
 
     /**
-     * Finds a programming exercise by its id including the submissions for its solution and template participation.
+     * Finds a {@link ProgrammingExercise} by its ID with optional dynamic fetching of associated entities.
      *
-     * @param exerciseId          The id of the programming exercise.
-     * @param withGradingCriteria True, if the grading instructions of the exercise should be included as well.
-     * @return A programming exercise that has the given id.
-     * @throws EntityNotFoundException In case no exercise with the given id exists.
+     * @param exerciseId   the ID of the programming exercise to find.
+     * @param fetchOptions a collection of {@link ProgrammingExerciseFetchOptions} indicating which associated entities to fetch.
+     * @return the {@link ProgrammingExercise} with the specified ID and the associated entities fetched according to the provided options.
+     * @throws EntityNotFoundException if the programming exercise with the specified ID does not exist.
      */
     @NotNull
-    default ProgrammingExercise findByIdWithAuxiliaryRepositoriesTeamAssignmentConfigAndGradingCriteriaElseThrow(long exerciseId, boolean withGradingCriteria)
-            throws EntityNotFoundException {
-
-        final Specification<ProgrammingExercise> specification = (root, query, criteriaBuilder) -> {
-            root.fetch(Exercise_.CATEGORIES, JoinType.LEFT);
-            root.fetch(Exercise_.TEAM_ASSIGNMENT_CONFIG, JoinType.LEFT);
-            root.fetch(ProgrammingExercise_.AUXILIARY_REPOSITORIES, JoinType.LEFT);
-
-            if (withGradingCriteria) {
-                root.fetch(Exercise_.GRADING_CRITERIA, JoinType.LEFT);
-            }
-
-            return null;
-        };
+    default ProgrammingExercise findByIdWithDynamicFetchElseThrow(long exerciseId, ProgrammingExerciseFetchOptions... fetchOptions) throws EntityNotFoundException {
+        return findByIdWithDynamicFetchElseThrow(exerciseId, Set.of(fetchOptions));
+    }
 
-        return findOneByIdElseThrow(specification, exerciseId);
+    /**
+     * Finds a {@link ProgrammingExercise} by its ID with optional dynamic fetching of associated entities.
+     *
+     * @param exerciseId   the ID of the programming exercise to find.
+     * @param fetchOptions a collection of {@link ProgrammingExerciseFetchOptions} indicating which associated entities to fetch.
+     * @return the {@link ProgrammingExercise} with the specified ID and the associated entities fetched according to the provided options.
+     * @throws EntityNotFoundException if the programming exercise with the specified ID does not exist.
+     */
+    @NotNull
+    default ProgrammingExercise findByIdWithDynamicFetchElseThrow(long exerciseId, Collection<ProgrammingExerciseFetchOptions> fetchOptions) throws EntityNotFoundException {
+        var specification = getDynamicSpecification(fetchOptions);
+        return findOneByIdElseThrow(specification, exerciseId, "Programming Exercise");
     }
 
     /**
@@ -852,4 +846,44 @@ default void generateBuildPlanAccessSecretIfNotExists(ProgrammingExercise exerci
             save(exercise);
         }
     }
+
+    /**
+     * Fetch options for the {@link ProgrammingExercise} entity.
+     * Each option specifies an entity or a collection of entities to fetch eagerly when using a dynamic fetching query.
+     */
+    enum ProgrammingExerciseFetchOptions implements FetchOptions {
+
+        // @formatter:off
+        Categories(Exercise_.CATEGORIES),
+        TeamAssignmentConfig(Exercise_.TEAM_ASSIGNMENT_CONFIG),
+        AuxiliaryRepositories(ProgrammingExercise_.AUXILIARY_REPOSITORIES),
+        GradingCriteria(Exercise_.GRADING_CRITERIA),
+        StudentParticipations(ProgrammingExercise_.STUDENT_PARTICIPATIONS),
+        TemplateParticipation(ProgrammingExercise_.TEMPLATE_PARTICIPATION),
+        SolutionParticipation(ProgrammingExercise_.SOLUTION_PARTICIPATION),
+        TestCases(ProgrammingExercise_.TEST_CASES),
+        Tasks(ProgrammingExercise_.TASKS),
+        StaticCodeAnalysisCategories(ProgrammingExercise_.STATIC_CODE_ANALYSIS_CATEGORIES),
+        SubmissionPolicy(ProgrammingExercise_.SUBMISSION_POLICY),
+        ExerciseHints(ProgrammingExercise_.EXERCISE_HINTS),
+        Competencies(ProgrammingExercise_.COMPETENCIES),
+        Teams(ProgrammingExercise_.TEAMS),
+        TutorParticipations(ProgrammingExercise_.TUTOR_PARTICIPATIONS),
+        ExampleSubmissions(ProgrammingExercise_.EXAMPLE_SUBMISSIONS),
+        Attachments(ProgrammingExercise_.ATTACHMENTS),
+        Posts(ProgrammingExercise_.POSTS),
+        PlagiarismCases(ProgrammingExercise_.PLAGIARISM_CASES),
+        PlagiarismDetectionConfig(ProgrammingExercise_.PLAGIARISM_DETECTION_CONFIG);
+        // @formatter:on
+
+        private final String fetchPath;
+
+        ProgrammingExerciseFetchOptions(String fetchPath) {
+            this.fetchPath = fetchPath;
+        }
+
+        public String getFetchPath() {
+            return fetchPath;
+        }
+    }
 }

src/main/java/de/tum/in/www1/artemis/repository/SolutionProgrammingExerciseParticipationRepository.java

@@ -1,26 +1,37 @@
 package de.tum.in.www1.artemis.repository;
 
 import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE;
+import static de.tum.in.www1.artemis.repository.SolutionProgrammingExerciseParticipationRepository.SolutionParticipationFetchOptions;
 import static org.springframework.data.jpa.repository.EntityGraph.EntityGraphType.LOAD;
 
+import java.util.Collection;
 import java.util.Optional;
 
+import jakarta.validation.constraints.NotNull;
+
 import org.springframework.context.annotation.Profile;
+import org.springframework.data.jpa.domain.Specification;
 import org.springframework.data.jpa.repository.EntityGraph;
-import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
 
+import de.tum.in.www1.artemis.domain.DomainObject_;
+import de.tum.in.www1.artemis.domain.Submission_;
 import de.tum.in.www1.artemis.domain.participation.SolutionProgrammingExerciseParticipation;
+import de.tum.in.www1.artemis.domain.participation.SolutionProgrammingExerciseParticipation_;
+import de.tum.in.www1.artemis.domain.participation.TemplateProgrammingExerciseParticipation_;
+import de.tum.in.www1.artemis.repository.base.DynamicSpecificationRepository;
+import de.tum.in.www1.artemis.repository.base.FetchOptions;
 import de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException;
 
 /**
  * Spring Data JPA repository for the Participation entity.
  */
 @Profile(PROFILE_CORE)
 @Repository
-public interface SolutionProgrammingExerciseParticipationRepository extends JpaRepository<SolutionProgrammingExerciseParticipation, Long> {
+public interface SolutionProgrammingExerciseParticipationRepository
+        extends DynamicSpecificationRepository<SolutionProgrammingExerciseParticipation, Long, SolutionParticipationFetchOptions> {
 
     @Query("""
             SELECT p
@@ -45,13 +56,55 @@ default SolutionProgrammingExerciseParticipation findWithEagerResultsAndSubmissi
     @EntityGraph(type = LOAD, attributePaths = { "submissions" })
     Optional<SolutionProgrammingExerciseParticipation> findWithEagerSubmissionsByProgrammingExerciseId(long exerciseId);
 
-    @EntityGraph(type = LOAD, attributePaths = { "submissions", "submissions.results" })
-    Optional<SolutionProgrammingExerciseParticipation> findWithEagerSubmissionsAndSubmissionResultsByProgrammingExerciseId(long exerciseId);
+    @NotNull
+    default SolutionProgrammingExerciseParticipation findByExerciseIdElseThrow(final Specification<SolutionProgrammingExerciseParticipation> specification, long exerciseId) {
+        final Specification<SolutionProgrammingExerciseParticipation> hasExerciseIdSpec = (root, query, criteriaBuilder) -> criteriaBuilder
+                .equal(root.get(TemplateProgrammingExerciseParticipation_.PROGRAMMING_EXERCISE).get(DomainObject_.ID), exerciseId);
+        return findOne(specification.and(hasExerciseIdSpec)).orElseThrow(() -> new EntityNotFoundException("Template Programming Exercise Participation --> Exercise", exerciseId));
+    }
+
+    /**
+     * Finds a {@link SolutionProgrammingExerciseParticipation} by the ID of its associated programming exercise with optional dynamic fetching of associated entities.
+     *
+     * @param exerciseId   the ID of the associated programming exercise.
+     * @param fetchOptions a collection of {@link SolutionParticipationFetchOptions} indicating which associated entities to fetch.
+     * @return the {@link SolutionProgrammingExerciseParticipation} associated with the specified programming exercise ID and the associated entities fetched according to the
+     *         provided options.
+     * @throws EntityNotFoundException if the solution programming exercise participation with the specified exercise ID does not exist.
+     */
+    @NotNull
+    default SolutionProgrammingExerciseParticipation findByExerciseIdWithDynamicFetchElseThrow(long exerciseId, Collection<SolutionParticipationFetchOptions> fetchOptions)
+            throws EntityNotFoundException {
+        var specification = getDynamicSpecification(fetchOptions);
+        return findByExerciseIdElseThrow(specification, exerciseId);
+    }
 
     Optional<SolutionProgrammingExerciseParticipation> findByProgrammingExerciseId(long programmingExerciseId);
 
     default SolutionProgrammingExerciseParticipation findByProgrammingExerciseIdElseThrow(long programmingExerciseId) {
         var optional = findByProgrammingExerciseId(programmingExerciseId);
         return optional.orElseThrow(() -> new EntityNotFoundException("Solution Programming Exercise Participation", programmingExerciseId));
     }
+
+    /**
+     * Fetch options for the {@link SolutionProgrammingExerciseParticipation} entity.
+     * Each option specifies an entity or a collection of entities to fetch eagerly when using a dynamic fetching query.
+     */
+    enum SolutionParticipationFetchOptions implements FetchOptions {
+
+        // @formatter:off
+        Submissions(SolutionProgrammingExerciseParticipation_.SUBMISSIONS),
+        SubmissionsAndResults(SolutionProgrammingExerciseParticipation_.SUBMISSIONS, Submission_.RESULTS);
+        // @formatter:on
+
+        private final String fetchPath;
+
+        SolutionParticipationFetchOptions(String... fetchPath) {
+            this.fetchPath = String.join(".", fetchPath);
+        }
+
+        public String getFetchPath() {
+            return fetchPath;
+        }
+    }
 }