diff --git a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java index 9b9aba3611f..502529db57b 100644 --- a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java +++ b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java @@ -40,7 +40,10 @@ public enum Features implements Feature { SEND_ALL_VERIFICATION_EMAILS, @Label("Send add works emails for 7, 28 and 90 days.") - SEND_ADD_WORKS_EMAILS; + SEND_ADD_WORKS_EMAILS, + + @Label("Delete events older than 90 days from the DB ") + DELETE_EVENTS; public boolean isActive() { return FeatureContext.getFeatureManager().isActive(this); diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java index 93108c62255..78a2bf775f6 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java @@ -2,6 +2,8 @@ import org.orcid.persistence.jpa.entities.EventEntity; +import java.util.List; + /** * * @author Daniel Palafox @@ -12,4 +14,10 @@ public interface EventDao { void createEvent(EventEntity eventEntity); EventEntity find(long id); + + void delete(long id); + + List findAll(); + + void deleteEventsByDate(Integer numberOfDays); } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java new file mode 100644 index 00000000000..c0c77b7730f --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java @@ -0,0 +1,17 @@ +package org.orcid.persistence.dao; + +import org.orcid.persistence.jpa.entities.EventStatsEntity; + +import java.util.List; + +/** + * + * @author Daniel Palafox + * + */ +public interface EventStatsDao { + + void createEventStats(); + + List findAll(); +} diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java index 6a07a45c018..e3e71561c4e 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java @@ -2,11 +2,15 @@ import javax.annotation.Resource; import javax.persistence.EntityManager; +import javax.persistence.Query; +import javax.persistence.TypedQuery; import org.orcid.persistence.dao.EventDao; import org.orcid.persistence.jpa.entities.EventEntity; import org.springframework.transaction.annotation.Transactional; +import java.util.List; + /** * @author Daniel Palafox */ @@ -29,5 +33,25 @@ public void createEvent(EventEntity eventEntity) { public EventEntity find(long id) { return entityManager.find(EventEntity.class, id); } - + + @Override + @Transactional + public void delete(long id) { + entityManager.remove(find(id)); + } + + @Override + public List findAll() { + TypedQuery query = entityManager.createQuery("from EventEntity", EventEntity.class); + return query.getResultList(); + } + + @Override + @Transactional + public void deleteEventsByDate(Integer numberOfDays) { + String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date)"; + Query queryDelete = entityManager.createNativeQuery(query); + queryDelete.setParameter("numberOfDays", numberOfDays); + queryDelete.executeUpdate(); + } } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java new file mode 100644 index 00000000000..2e68e076ad5 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java @@ -0,0 +1,40 @@ +package org.orcid.persistence.dao.impl; + +import org.orcid.persistence.dao.EventStatsDao; +import org.orcid.persistence.jpa.entities.EventStatsEntity; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import javax.persistence.EntityManager; +import javax.persistence.Query; +import javax.persistence.TypedQuery; +import java.util.List; + +/** + * @author Daniel Palafox + */ +public class EventStatsDaoImpl implements EventStatsDao { + + @Resource(name = "entityManager") + protected EntityManager entityManager; + + @Override + @Transactional + public void createEventStats() { + String query = + "INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) " + + "SELECT event_type, client_id, COUNT(id), CAST(e.date_created as date), now(), now() " + + "FROM event as e " + + "WHERE CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " + + "GROUP BY event_type, client_id, CAST(e.date_created as date) " + + "ORDER BY CAST(e.date_created as date) DESC;"; + Query insertQuery = entityManager.createNativeQuery(query); + insertQuery.executeUpdate(); + } + + @Override + public List findAll() { + TypedQuery query = entityManager.createQuery("from EventStatsEntity", EventStatsEntity.class); + return query.getResultList(); + } +} diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventStatsEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventStatsEntity.java new file mode 100644 index 00000000000..9134bdaa496 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventStatsEntity.java @@ -0,0 +1,92 @@ +package org.orcid.persistence.jpa.entities; + +import java.util.Date; +import java.util.Objects; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; + +/** + * + * @author Daniel Palafox + * + */ +@Entity +@Table(name = "event_stats") +public class EventStatsEntity extends BaseEntity{ + private Long id; + private String eventType; + private String clientId; + private Integer count; + private Date date; + + @Id + @Column(name = "id") + @GeneratedValue(strategy = GenerationType.AUTO, generator = "event_seq") + @SequenceGenerator(name = "event_seq", sequenceName = "event_seq", allocationSize = 1) + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Column(name = "event_type") + public String getEventType() { + return eventType; + } + + public void setEventType(String eventType) { + this.eventType = eventType; + } + + @Column(name = "client_id") + public String getClientId() { + return clientId; + } + + public void setClientId(String client_id) { + this.clientId = client_id; + } + + @Column(name = "count") + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } + + @Column(name = "date") + public Date getDate() { + return date; + } + + public void setDate(Date date) { + this.date = date; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EventStatsEntity that = (EventStatsEntity) o; + return Objects.equals(getId(), that.getId()) && + Objects.equals(getEventType(), that.getEventType()) && + Objects.equals(getClientId(), that.getClientId()) && + Objects.equals(getCount(), that.getCount()) && + Objects.equals(getDate(), that.getDate()); + } + + @Override + public int hashCode() { + return Objects.hash(getId(), getEventType(), getClientId(), getCount(), getDate()); + } +} diff --git a/orcid-persistence/src/main/resources/META-INF/persistence.xml b/orcid-persistence/src/main/resources/META-INF/persistence.xml index c3af43f7ae1..2ef97de7a61 100644 --- a/orcid-persistence/src/main/resources/META-INF/persistence.xml +++ b/orcid-persistence/src/main/resources/META-INF/persistence.xml @@ -61,6 +61,7 @@ org.orcid.persistence.jpa.entities.FindMyStuffHistoryEntity org.orcid.persistence.jpa.entities.SpamEntity org.orcid.persistence.jpa.entities.EventEntity + org.orcid.persistence.jpa.entities.EventStatsEntity org.orcid.persistence.jpa.entities.EmailDomainEntity diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index c27b11c2f22..b168e83cc29 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -381,4 +381,6 @@ + + diff --git a/orcid-persistence/src/main/resources/db/updates/create_event_stats_table.xml b/orcid-persistence/src/main/resources/db/updates/create_event_stats_table.xml new file mode 100644 index 00000000000..397d855fdaf --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/create_event_stats_table.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SELECT 1 FROM pg_roles WHERE rolname='orcidro' + + GRANT SELECT ON event_stats to orcidro; + + + diff --git a/orcid-persistence/src/main/resources/db/updates/dw_event_stats.xml b/orcid-persistence/src/main/resources/db/updates/dw_event_stats.xml new file mode 100644 index 00000000000..ff98617794b --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/dw_event_stats.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified + FROM event_stats + ORDER BY DATE_TRUNC('day', date_created) DESC; + + + + + + SELECT 1 FROM pg_roles WHERE rolname='dw_user' + + GRANT SELECT ON TABLE dw_event_stats to dw_user; + + + diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 2d6752cb261..7110cf1484a 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -459,7 +459,8 @@ - + + diff --git a/orcid-persistence/src/test/java/org/orcid/persistence/EventDaoTest.java b/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java similarity index 81% rename from orcid-persistence/src/test/java/org/orcid/persistence/EventDaoTest.java rename to orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java index a48fd69fea6..025c8b0d440 100644 --- a/orcid-persistence/src/test/java/org/orcid/persistence/EventDaoTest.java +++ b/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java @@ -1,4 +1,4 @@ -package org.orcid.persistence; +package org.orcid.persistence.dao; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -6,15 +6,16 @@ import java.util.Arrays; import java.util.Date; +import java.util.List; import javax.annotation.Resource; +import javax.persistence.EntityManager; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.orcid.persistence.dao.EventDao; import org.orcid.persistence.jpa.entities.EventEntity; import org.orcid.test.DBUnitTest; import org.orcid.test.OrcidJUnit4ClassRunner; @@ -39,6 +40,21 @@ public static void removeDBUnitData() throws Exception { removeDBUnitData(Arrays.asList("/data/EventEntityData.xml", "/data/ProfileEntityData.xml", "/data/SourceClientDetailsEntityData.xml")); } + @Test + public void deleteEventsByDate() { + List eventEntityList = eventDao.findAll(); + + assertNotNull(eventEntityList); + assertEquals(3, eventEntityList.size()); + + eventDao.deleteEventsByDate(90); + + eventEntityList = eventDao.findAll(); + + assertNotNull(eventEntityList); + assertEquals(0, eventEntityList.size()); + } + @Test public void testWriteEvent() throws IllegalAccessException { EventEntity eventEntity = new EventEntity(); @@ -62,6 +78,7 @@ public void testWriteEvent() throws IllegalAccessException { assertEquals(eventEntity.getId(), fromDb.getId()); assertEquals(eventEntity.getLabel(), fromDb.getLabel()); assertNotNull(fromDb.getDateCreated()); + + eventDao.delete(eventEntity.getId()); } - } diff --git a/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventStatsDaoTest.java b/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventStatsDaoTest.java new file mode 100644 index 00000000000..780f294d37a --- /dev/null +++ b/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventStatsDaoTest.java @@ -0,0 +1,55 @@ +package org.orcid.persistence.dao; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.orcid.persistence.jpa.entities.EventEntity; +import org.orcid.persistence.jpa.entities.EventStatsEntity; +import org.orcid.test.OrcidJUnit4ClassRunner; +import org.springframework.test.context.ContextConfiguration; + +import javax.annotation.Resource; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +@RunWith(OrcidJUnit4ClassRunner.class) +@ContextConfiguration(inheritInitializers = false, inheritLocations = false, locations = {"classpath:test-orcid-persistence-context.xml"}) +public class EventStatsDaoTest { + + @Resource(name = "eventDao") + private EventDao eventDao; + + @Resource(name = "eventStatsDao") + private EventStatsDao eventStatsDao; + + @Test + public void createEventStats() { + createEvents(); + + eventStatsDao.createEventStats(); + + List eventStatsEntityList = eventStatsDao.findAll(); + + assertNotNull(eventStatsEntityList); + assertEquals(1, eventStatsEntityList.size()); + assertEquals(Integer.valueOf(20), eventStatsEntityList.get(0).getCount()); + } + + private void createEvents() { + for (int i = 0; i < 20; i++) { + EventEntity eventEntity = new EventEntity(); + eventEntity.setEventType("Sign-In"); + eventEntity.setClientId("Client " + 1); + LocalDate date = LocalDate.now().minusDays(1); + Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); + eventEntity.setDateCreated(Date.from(instant)); + eventDao.createEvent(eventEntity); + } + } + +} diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/web/event/EventStats.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/web/event/EventStats.java new file mode 100644 index 00000000000..91d6a941e82 --- /dev/null +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/web/event/EventStats.java @@ -0,0 +1,8 @@ +package org.orcid.scheduler.web.event; + +public interface EventStats { + + void saveEventStats(); + + void deleteEvents(); +} diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/web/event/impl/EventStatsImpl.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/web/event/impl/EventStatsImpl.java new file mode 100644 index 00000000000..304d8c47c2a --- /dev/null +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/web/event/impl/EventStatsImpl.java @@ -0,0 +1,42 @@ +package org.orcid.scheduler.web.event.impl; + +import org.orcid.core.togglz.Features; +import org.orcid.persistence.dao.EventDao; +import org.orcid.persistence.dao.EventStatsDao; +import org.orcid.scheduler.web.event.EventStats; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; + +import javax.annotation.Resource; +import java.time.LocalDate; + +public class EventStatsImpl implements EventStats { + + private static final Logger LOGGER = LoggerFactory.getLogger(EventStatsImpl.class); + + @Resource(name = "eventDao") + private EventDao eventDao; + + @Resource(name = "eventStatsDao") + private EventStatsDao eventStatsDao; + + @Value("${org.orcid.scheduler.event.deleteEvents.numberOfDays:90}") + private int DELETE_EVENTS_OLDER_THAN_DAYS; + + @Override + public void saveEventStats() { + LocalDate date = LocalDate.now().minusDays(1); + String currentDate = date.getDayOfMonth() + "/" + date.getMonth() + "/" + date.getYear(); + LOGGER.debug("Storing aggregate data to event_stats table of the day" + currentDate); + eventStatsDao.createEventStats(); + } + + @Override + public void deleteEvents() { + if (Features.DELETE_EVENTS.isActive()) { + LOGGER.debug("Deleting events older than "+ DELETE_EVENTS_OLDER_THAN_DAYS +" days"); + eventDao.deleteEventsByDate(DELETE_EVENTS_OLDER_THAN_DAYS); + } + } +} diff --git a/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml b/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml index d5933d38a2c..11b7dec2c5c 100644 --- a/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml +++ b/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml @@ -41,6 +41,8 @@ + + @@ -132,4 +134,7 @@ + + + diff --git a/orcid-test/src/main/resources/data/EventEntityData.xml b/orcid-test/src/main/resources/data/EventEntityData.xml index 9ac989cc1f1..83aa14b811b 100644 --- a/orcid-test/src/main/resources/data/EventEntityData.xml +++ b/orcid-test/src/main/resources/data/EventEntityData.xml @@ -13,7 +13,7 @@ />