VKT(Backend): OPHVKTKEH-111 ilmoittautumisten anonymisointi 180pv tutkintopäivän jälkeen

backend/vkt/db/1_tables.sql

@@ -3,7 +3,7 @@
 --
 
 -- Dumped from database version 12.9 (Debian 12.9-1.pgdg110+1)
--- Dumped by pg_dump version 14.7 (Homebrew)
+-- Dumped by pg_dump version 12.9 (Debian 12.9-1.pgdg110+1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -139,7 +139,8 @@ CREATE TABLE public.enrollment (
     town text,
     country text,
     payment_link_hash character varying(255),
-    payment_link_expires_at timestamp with time zone
+    payment_link_expires_at timestamp with time zone,
+    is_anonymized boolean NOT NULL
 );
 
 
@@ -250,7 +251,8 @@ CREATE TABLE public.payment (
     transaction_id text,
     reference text,
     payment_url text,
-    payment_status text
+    payment_status text,
+    refunded_at timestamp with time zone
 );
 
 

backend/vkt/db/2_tables_data.sql

@@ -3,7 +3,7 @@
 --
 
 -- Dumped from database version 12.9 (Debian 12.9-1.pgdg110+1)
--- Dumped by pg_dump version 14.7 (Homebrew)
+-- Dumped by pg_dump version 12.9 (Debian 12.9-1.pgdg110+1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;

backend/vkt/db/3_liquibase.sql

@@ -3,7 +3,7 @@
 --
 
 -- Dumped from database version 12.9 (Debian 12.9-1.pgdg110+1)
--- Dumped by pg_dump version 14.7 (Homebrew)
+-- Dumped by pg_dump version 12.9 (Debian 12.9-1.pgdg110+1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -77,17 +77,19 @@ COPY public.databasechangelog (id, author, filename, dateexecuted, orderexecuted
 2022-12-06-add-enum-email_type	terova	migrations.xml	2022-12-06 14:21:45.065383	12	EXECUTED	8:9d2dd6c5fb47e67ba50c6cf0db4edd47	createTable tableName=email_type; insert tableName=email_type		\N	4.9.1	\N	\N	0336504848
 2022-12-06-create-email-table	terova	migrations.xml	2022-12-06 14:21:45.096787	13	EXECUTED	8:fc290ff4700b729ac568057c7dd6c211	createTable tableName=email; addForeignKeyConstraint baseTableName=email, constraintName=fk_email_email_type, referencedTableName=email_type		\N	4.9.1	\N	\N	0336504848
 2022-12-06-create-email_attachment-table	terova	migrations.xml	2022-12-06 18:42:00.87481	14	EXECUTED	8:27ead2667c986a4fb6325d9d93238151	createTable tableName=email_attachment; addForeignKeyConstraint baseTableName=email_attachment, constraintName=fk_email_attachment_email, referencedTableName=email		\N	4.9.1	\N	\N	0352120682
-2023-01-18-modify_enrollment-table_previous-enrollment-date	mikhuttu	migrations.xml	2023-05-29 09:17:57.498963	15	EXECUTED	8:b5bb9828cdc9f6349c1e92af6da50bcd	modifyDataType columnName=previous_enrollment_date, tableName=enrollment; renameColumn newColumnName=previous_enrollment, oldColumnName=previous_enrollment_date, tableName=enrollment		\N	4.9.1	\N	\N	5351877372
-2023-02-03-add_reservation-table_renewed_at	jrkkp	migrations.xml	2023-05-29 09:17:57.503008	16	EXECUTED	8:e9cd840a8006f4a136fac6e259048628	addColumn tableName=reservation		\N	4.9.1	\N	\N	5351877372
-2023-03-20-add_spring_session_table	jrkkp	migrations.xml	2023-05-29 09:17:57.520006	17	EXECUTED	8:74529b81aca9ec770312c7017e0c594d	createTable tableName=spring_session; createIndex indexName=spring_session_expires_idx, tableName=spring_session; createIndex indexName=spring_session_principal_idx, tableName=spring_session; createTable tableName=spring_session_attributes; addPri...		\N	4.9.1	\N	\N	5351877372
-2023-04-11-add_person_oid	jrkkp	migrations.xml	2023-05-29 09:17:57.530923	18	EXECUTED	8:5b0623c9012a91e8f39b129672804bde	addColumn tableName=person; dropNotNullConstraint columnName=identity_number, tableName=person		\N	4.9.1	\N	\N	5351877372
-2023-05-03-add-enrollment-to-queue-confirmation-email_type	mikhuttu	migrations.xml	2023-05-29 09:17:57.534001	19	EXECUTED	8:4502d8b0afd0a1b88b7649fa2db135f4	insert tableName=email_type		\N	4.9.1	\N	\N	5351877372
-2023-05-25-payment-table	jrkkp	migrations.xml	2023-05-29 09:17:57.545296	20	EXECUTED	8:1869b55cb1a464dee967a5b832c80003	createTable tableName=payment; insert tableName=enrollment_status; insert tableName=enrollment_status; addForeignKeyConstraint baseTableName=payment, constraintName=fk_payment_enrollment, referencedTableName=enrollment		\N	4.9.1	\N	\N	5351877372
-2023-05-30-modify_payment-table_amount	mikhuttu	migrations.xml	2023-05-30 13:31:02.784706	21	EXECUTED	8:a1e8d1377da2bc6360f9971c8b052cb4	modifyDataType columnName=amount, tableName=payment		\N	4.9.1	\N	\N	5453462712
-2023-06-01-enrollment-payment-link-hash	jrkkp	migrations.xml	2023-06-01 13:53:01.472105	22	EXECUTED	8:0fc928a86fa41527372d8e8af21e813b	addColumn tableName=enrollment		\N	4.9.1	\N	\N	5627581378
-2023-06-02-rename-enrollment-status-EXPECTING_PAYMENT	mikhuttu	migrations.xml	2023-06-02 08:38:57.704377	23	EXECUTED	8:a35dc4c9e0d0d241f4293f0b2ba16224	insert tableName=enrollment_status; sql; sql		\N	4.9.1	\N	\N	5695137627
-2023-06-16-person-latest-identified-at	mikhuttu	migrations.xml	2023-06-16 09:46:36.462511	24	EXECUTED	8:51d5c16082a3fd83108e9c40e7ae78e6	addColumn tableName=person; sql; addNotNullConstraint columnName=latest_identified_at, tableName=person		\N	4.20.0	\N	\N	6908796433
-2023-06-29-remove-person-identity_number	mikhuttu	migrations.xml	2023-06-29 09:32:48.572192	25	EXECUTED	8:5b23bce4f54b5583b757ad6bb81c612d	dropColumn columnName=identity_number, tableName=person; dropColumn columnName=date_of_birth, tableName=person		\N	4.20.0	\N	\N	8031168558
+2023-01-18-modify_enrollment-table_previous-enrollment-date	mikhuttu	migrations.xml	2023-10-05 12:40:24.529359	15	EXECUTED	8:b5bb9828cdc9f6349c1e92af6da50bcd	modifyDataType columnName=previous_enrollment_date, tableName=enrollment; renameColumn newColumnName=previous_enrollment, oldColumnName=previous_enrollment_date, tableName=enrollment		\N	4.20.0	\N	\N	6498824505
+2023-02-03-add_reservation-table_renewed_at	jrkkp	migrations.xml	2023-10-05 12:40:24.534427	16	EXECUTED	8:e9cd840a8006f4a136fac6e259048628	addColumn tableName=reservation		\N	4.20.0	\N	\N	6498824505
+2023-03-20-add_spring_session_table	jrkkp	migrations.xml	2023-10-05 12:40:24.548449	17	EXECUTED	8:74529b81aca9ec770312c7017e0c594d	createTable tableName=spring_session; createIndex indexName=spring_session_expires_idx, tableName=spring_session; createIndex indexName=spring_session_principal_idx, tableName=spring_session; createTable tableName=spring_session_attributes; addPri...		\N	4.20.0	\N	\N	6498824505
+2023-04-11-add_person_oid	jrkkp	migrations.xml	2023-10-05 12:40:24.558025	18	EXECUTED	8:5b0623c9012a91e8f39b129672804bde	addColumn tableName=person; dropNotNullConstraint columnName=identity_number, tableName=person		\N	4.20.0	\N	\N	6498824505
+2023-05-03-add-enrollment-to-queue-confirmation-email_type	mikhuttu	migrations.xml	2023-10-05 12:40:24.561463	19	EXECUTED	8:4502d8b0afd0a1b88b7649fa2db135f4	insert tableName=email_type		\N	4.20.0	\N	\N	6498824505
+2023-05-25-payment-table	jrkkp	migrations.xml	2023-10-05 12:40:24.571098	20	EXECUTED	8:1869b55cb1a464dee967a5b832c80003	createTable tableName=payment; insert tableName=enrollment_status; insert tableName=enrollment_status; addForeignKeyConstraint baseTableName=payment, constraintName=fk_payment_enrollment, referencedTableName=enrollment		\N	4.20.0	\N	\N	6498824505
+2023-05-30-modify_payment-table_amount	mikhuttu	migrations.xml	2023-10-05 12:40:24.577595	21	EXECUTED	8:a1e8d1377da2bc6360f9971c8b052cb4	modifyDataType columnName=amount, tableName=payment		\N	4.20.0	\N	\N	6498824505
+2023-06-01-enrollment-payment-link-hash	jrkkp	migrations.xml	2023-10-05 12:40:24.583222	22	EXECUTED	8:0fc928a86fa41527372d8e8af21e813b	addColumn tableName=enrollment		\N	4.20.0	\N	\N	6498824505
+2023-06-02-rename-enrollment-status-EXPECTING_PAYMENT	mikhuttu	migrations.xml	2023-10-05 12:40:24.591858	23	EXECUTED	8:a35dc4c9e0d0d241f4293f0b2ba16224	insert tableName=enrollment_status; sql; sql		\N	4.20.0	\N	\N	6498824505
+2023-06-16-person-latest-identified-at	mikhuttu	migrations.xml	2023-10-05 12:40:24.596771	24	EXECUTED	8:51d5c16082a3fd83108e9c40e7ae78e6	addColumn tableName=person; sql; addNotNullConstraint columnName=latest_identified_at, tableName=person		\N	4.20.0	\N	\N	6498824505
+2023-06-29-remove-person-identity_number	mikhuttu	migrations.xml	2023-10-05 12:40:24.600916	25	EXECUTED	8:5b23bce4f54b5583b757ad6bb81c612d	dropColumn columnName=identity_number, tableName=person; dropColumn columnName=date_of_birth, tableName=person		\N	4.20.0	\N	\N	6498824505
+2023-08-03-payment-add-refunded	jrkkp	migrations.xml	2023-10-05 12:40:24.60417	26	EXECUTED	8:fb56e2324ab73d52aefd19c907295aa3	addColumn tableName=payment		\N	4.20.0	\N	\N	6498824505
+2023-09-07-enrollment-is-anonymized	mikhuttu	migrations.xml	2023-10-05 12:40:24.608176	27	EXECUTED	8:3a3c0039e1c2c3e5a79b4e62612246c1	addColumn tableName=enrollment; dropDefaultValue columnName=is_anonymized, tableName=enrollment		\N	4.20.0	\N	\N	6498824505
 \.
 
 

backend/vkt/db/4_init.sql

@@ -191,7 +191,7 @@ FROM generate_series(1, 22) i,
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1 OFFSET 1), person_id,
        true, true, true,
        true, true, true, true,
@@ -209,7 +209,8 @@ SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1
            END,
        CASE mod(person_id, 5)
            WHEN 0 THEN countries[mod(person_id / 5 - 1, array_length(countries, 1)) + 1]
-           END
+           END,
+       false
 FROM person,
      (SELECT ('{Erottajankatu 1, Mäkelänkatu 70, Postikatu 2, Hamngatan 4}')::text[] AS streets) AS street_table,
      (SELECT ('{00130, 00610, 33100, 111 47}')::text[] AS postal_codes) AS postal_code_table,
@@ -221,7 +222,7 @@ ORDER BY person_id LIMIT (SELECT max_participants FROM exam_event ORDER BY exam_
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1 OFFSET 2), person_id,
        true, true, true,
        true, true, true, true,
@@ -239,7 +240,8 @@ SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1
            END,
        CASE mod(person_id, 5)
            WHEN 0 THEN countries[mod(person_id / 5 - 1, array_length(countries, 1)) + 1]
-           END
+           END,
+       false
 FROM person,
      (SELECT ('{Erottajankatu 1, Mäkelänkatu 70, Postikatu 2, Hamngatan 4}')::text[] AS streets) AS street_table,
      (SELECT ('{00130, 00610, 33100, 111 47}')::text[] AS postal_codes) AS postal_code_table,
@@ -249,7 +251,7 @@ ORDER BY person_id LIMIT (SELECT max_participants FROM exam_event ORDER BY exam_
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1 OFFSET 2), person_id,
        true, true, true,
        true, true, true, true,
@@ -267,7 +269,8 @@ SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1
            END,
        CASE mod(person_id, 5)
            WHEN 0 THEN countries[mod(person_id / 5 - 1, array_length(countries, 1)) + 1]
-           END
+           END,
+       false
 FROM person,
      (SELECT ('{Erottajankatu 1, Mäkelänkatu 70, Postikatu 2, Hamngatan 4}')::text[] AS streets) AS street_table,
      (SELECT ('{00130, 00610, 33100, 111 47}')::text[] AS postal_codes) AS postal_code_table,
@@ -281,7 +284,7 @@ ORDER BY person_id
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1 OFFSET 3), person_id,
        true, true, true,
        true, true, true, true,
@@ -299,7 +302,8 @@ SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1
            END,
        CASE mod(person_id, 5)
            WHEN 0 THEN countries[mod(person_id / 5 - 1, array_length(countries, 1)) + 1]
-           END
+           END,
+      false
 FROM person,
      (SELECT ('{Erottajankatu 1, Mäkelänkatu 70, Postikatu 2, Hamngatan 4}')::text[] AS streets) AS street_table,
      (SELECT ('{00130, 00610, 33100, 111 47}')::text[] AS postal_codes) AS postal_code_table,
@@ -308,7 +312,7 @@ FROM person,
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1 OFFSET 3), person_id,
        true, true, true,
        true, true, true, true,
@@ -326,7 +330,8 @@ SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1
            END,
        CASE mod(person_id, 5)
            WHEN 0 THEN countries[mod(person_id / 5 - 1, array_length(countries, 1)) + 1]
-           END
+           END,
+       false
 FROM person,
      (SELECT ('{Erottajankatu 1, Mäkelänkatu 70, Postikatu 2, Hamngatan 4}')::text[] AS streets) AS street_table,
      (SELECT ('{00130, 00610, 33100, 111 47}')::text[] AS postal_codes) AS postal_code_table,
@@ -339,7 +344,7 @@ FROM person,
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1 OFFSET 4), person_id,
        true, true, true,
        true, true, true, true,
@@ -357,7 +362,8 @@ SELECT (SELECT exam_event_id FROM exam_event ORDER BY exam_event_id DESC LIMIT 1
            END,
        CASE mod(person_id, 5)
            WHEN 0 THEN countries[mod(person_id / 5 - 1, array_length(countries, 1)) + 1]
-           END
+           END,
+       false
 FROM person,
      (SELECT ('{Erottajankatu 1, Mäkelänkatu 70, Postikatu 2, Hamngatan 4}')::text[] AS streets) AS street_table,
      (SELECT ('{00130, 00610, 33100, 111 47}')::text[] AS postal_codes) AS postal_code_table,
@@ -368,10 +374,10 @@ FROM person,
 INSERT INTO enrollment(exam_event_id, person_id,
                        skill_oral, skill_textual, skill_understanding,
                        partial_exam_speaking, partial_exam_speech_comprehension, partial_exam_writing, partial_exam_reading_comprehension,
-                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country)
+                       status, digital_certificate_consent, email, phone_number, street, postal_code, town, country, is_anonymized)
 SELECT exam_event_id, (SELECT max(person_id) FROM person),
        true, true, true,
        true, true, true, true,
        'CANCELED', true,
-       '[email protected]', '0404040404', null, null, null, null
+       '[email protected]', '0404040404', null, null, null, null, false
 FROM exam_event;

backend/vkt/src/main/java/fi/oph/vkt/config/Constants.java

@@ -14,4 +14,6 @@ public class Constants {
   public static final String DELETE_EXPIRED_RESERVATIONS_CRON = "0 30 9 * * *";
   // Daily at 10:00
   public static final String DELETE_OBSOLETE_PERSONS_CRON = "0 0 10 * * *";
+  // Daily at 10:30
+  public static final String ANONYMIZE_ENROLLMENTS_CRON = "0 30 10 * * *";
 }

backend/vkt/src/main/java/fi/oph/vkt/model/Enrollment.java

@@ -87,6 +87,9 @@ public class Enrollment extends BaseEntity {
   @Column(name = "payment_link_expires_at")
   private LocalDateTime paymentLinkExpiresAt;
 
+  @Column(name = "is_anonymized", nullable = false)
+  private boolean isAnonymized;
+
   @ManyToOne(fetch = FetchType.LAZY, optional = false)
   @JoinColumn(name = "exam_event_id", referencedColumnName = "exam_event_id", nullable = false)
   private ExamEvent examEvent;

backend/vkt/src/main/java/fi/oph/vkt/repository/EnrollmentRepository.java

@@ -4,12 +4,17 @@
 import fi.oph.vkt.model.ExamEvent;
 import fi.oph.vkt.model.Person;
 import fi.oph.vkt.model.type.EnrollmentStatus;
+import java.time.LocalDate;
 import java.util.List;
 import java.util.Optional;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
 
 @Repository
 public interface EnrollmentRepository extends BaseRepository<Enrollment> {
+  @Query("SELECT e FROM Enrollment e JOIN e.examEvent ee WHERE ee.date < ?1 AND NOT e.isAnonymized")
+  List<Enrollment> findAllToAnonymize(final LocalDate examDateBefore);
+
   List<Enrollment> findAllByStatus(final EnrollmentStatus enrollmentStatus);
   Optional<Enrollment> findByExamEventAndPerson(final ExamEvent examEvent, final Person person);
   Optional<Enrollment> findByExamEventAndPaymentLinkHash(final ExamEvent examEvent, final String paymentLinkHash);

backend/vkt/src/main/java/fi/oph/vkt/scheduled/AnonymizeEnrollments.java

@@ -0,0 +1,36 @@
+package fi.oph.vkt.scheduled;
+
+import fi.oph.vkt.config.Constants;
+import fi.oph.vkt.service.ClerkEnrollmentService;
+import fi.oph.vkt.util.SchedulingUtil;
+import jakarta.annotation.Resource;
+import lombok.RequiredArgsConstructor;
+import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class AnonymizeEnrollments {
+
+  private static final Logger LOG = LoggerFactory.getLogger(AnonymizeEnrollments.class);
+
+  private static final String LOCK_AT_LEAST = "PT1S";
+
+  private static final String LOCK_AT_MOST = "PT1H";
+
+  @Resource
+  private final ClerkEnrollmentService clerkEnrollmentService;
+
+  @Scheduled(cron = Constants.ANONYMIZE_ENROLLMENTS_CRON)
+  @SchedulerLock(name = "anonymizeEnrollments", lockAtLeastFor = LOCK_AT_LEAST, lockAtMostFor = LOCK_AT_MOST)
+  public void action() {
+    SchedulingUtil.runWithScheduledUser(() -> {
+      LOG.info("anonymizeEnrollments");
+
+      clerkEnrollmentService.anonymizeEnrollments();
+    });
+  }
+}