Skip to content

Commit

Permalink
Refactor fulltext search (#2052)
Browse files Browse the repository at this point in the history 
Save fulltext using DBAL instead of flush

Fixes a bug that called flush() exponentially during Doctrine lifecycle events
  • Loading branch information
@jimsafley
jimsafley authored May 24, 2023
1 parent d3b886b commit 6d1cae1
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 58 deletions.
73 changes: 45 additions & 28 deletions application/Module.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

use Omeka\Api\Adapter\FulltextSearchableInterface;
use Omeka\Api\Representation\AbstractResourceEntityRepresentation;
use Omeka\Entity\Item;
use Omeka\Entity\Media;
use Omeka\Module\AbstractModule;
use Laminas\EventManager\Event as ZendEvent;
Expand Down Expand Up @@ -121,20 +122,14 @@ public function attachListeners(SharedEventManagerInterface $sharedEventManager)

$sharedEventManager->attach(
'Omeka\Entity\Media',
'entity.persist.post',
[$this, 'saveFulltextOnMediaSave']
);

$sharedEventManager->attach(
'Omeka\Entity\Media',
'entity.update.post',
[$this, 'saveFulltextOnMediaSave']
'entity.remove.pre',
[$this, 'deleteFulltextMedia']
);

$sharedEventManager->attach(
'Omeka\Api\Adapter\SitePageAdapter',
'api.delete.pre',
[$this, 'deleteFulltextPre']
[$this, 'deleteFulltextPreSitePage']
);

$sharedEventManager->attach(
Expand Down Expand Up @@ -564,34 +559,42 @@ public function saveFulltext(ZendEvent $event)
{
$adapter = $event->getTarget();
$entity = $event->getParam('response')->getContent();
$fulltextSearch = $this->getServiceLocator()->get('Omeka\FulltextSearch');
$fulltextSearch->save($entity, $adapter);

// Item create needs special handling. We must save media fulltext here
// because media is created via cascade persist (during item create/update),
// which is invisible to normal API events.
if ($entity instanceof Item) {
$mediaAdapter = $adapter->getAdapter('media');
foreach ($entity->getMedia() as $mediaEntity) {
$fulltextSearch->save($mediaEntity, $mediaAdapter);
}
}
// Item media needs special handling. We must update the item's fulltext
// to append updated media data.
if ($entity instanceof Media) {
// Media get special handling during entity.persist.post and
// entity.update.post in self::saveFulltextOnMediaSave(). There's no
// need to process them here.
return;
$itemEntity = $entity->getItem();
$itemAdapter = $adapter->getAdapter('items');
$fulltextSearch->save($itemEntity, $itemAdapter);
}
$fulltext = $this->getServiceLocator()->get('Omeka\FulltextSearch');
$fulltext->save($entity, $adapter);
}

/**
* Save fulltext on media save.
* Delete the fulltext of a media.
*
* This method does two things. First, it updates the parent item's fulltext
* to contain any new text introduced by this media. Second it ensures that
* the fulltext of newly created media is saved. Otherwise, media created
* in the item context (via cascade persist) will not have fulltext.
* We must delete media fulltext here because media may be deleted via cascade
* remove (during item update), which is invisible to normal API events.
*
* @param ZendEvent $event
*/
public function saveFulltextOnMediaSave(ZendEvent $event)
public function deleteFulltextMedia(ZendEvent $event)
{
$fulltextSearch = $this->getServiceLocator()->get('Omeka\FulltextSearch');
$adapterManager = $this->getServiceLocator()->get('Omeka\ApiAdapterManager');
$mediaEntity = $event->getTarget();
$itemEntity = $mediaEntity->getItem();
$fulltextSearch->save($mediaEntity, $adapterManager->get('media'));
$fulltextSearch->save($itemEntity, $adapterManager->get('items'));
$mediaAdapter = $adapterManager->get('media');
$fulltextSearch->delete($mediaEntity->getId(), $mediaAdapter);
}

/**
Expand All @@ -603,7 +606,7 @@ public function saveFulltextOnMediaSave(ZendEvent $event)
*
* @param ZendEvent $event
*/
public function deleteFulltextPre(ZendEvent $event)
public function deleteFulltextPreSitePage(ZendEvent $event)
{
$request = $event->getParam('request');
$em = $this->getServiceLocator()->get('Omeka\EntityManager');
Expand All @@ -624,10 +627,24 @@ public function deleteFulltextPre(ZendEvent $event)
*/
public function deleteFulltext(ZendEvent $event)
{
$fulltext = $this->getServiceLocator()->get('Omeka\FulltextSearch');
$adapter = $event->getTarget();
$entity = $event->getParam('response')->getContent();
$request = $event->getParam('request');
$fulltext->delete(
// Note that the resource may not have an ID after being deleted.
$fulltextSearch = $this->getServiceLocator()->get('Omeka\FulltextSearch');

// Media delete needs special handling. We must update the item's fulltext
// to remove the appended media data. We return here because deleting media
// fulltext is handled by self::deleteFulltextMedia().
if ($entity instanceof Media) {
$itemEntity = $entity->getItem();
$itemAdapter = $adapter->getAdapter('items');
$fulltextSearch->save($itemEntity, $itemAdapter);
return;
}

// Note that the resource may not have an ID after being deleted. This
// is why we must use $request->getId() rather than $entity->getId().
$fulltextSearch->delete(
$request->getOption('deleted_entity_id') ?? $request->getId(),
$event->getTarget()
);
Expand Down
14 changes: 13 additions & 1 deletion application/src/Job/IndexFulltextSearch.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
namespace Omeka\Job;

use Omeka\Api\Adapter\FulltextSearchableInterface;
use Omeka\Api\Adapter\ResourceAdapter;
use Omeka\Api\Adapter\ValueAnnotationAdapter;

class IndexFulltextSearch extends AbstractJob
Expand All @@ -14,12 +15,23 @@ public function perform()
$services = $this->getServiceLocator();
$api = $services->get('Omeka\ApiManager');
$em = $services->get('Omeka\EntityManager');
$conn = $services->get('Omeka\Connection');
$fulltext = $services->get('Omeka\FulltextSearch');
$adapters = $services->get('Omeka\ApiAdapterManager');

// First delete all rows from the fulltext table to clear out the
// resources that don't belong.
$conn->executeStatement('DELETE FROM `fulltext_search`');

// Then iterate through all resource types and index the ones that are
// fulltext searchable. Note that we don't index "resource" and "value
// annotation" resources.
foreach ($adapters->getRegisteredNames() as $adapterName) {
$adapter = $adapters->get($adapterName);
if ($adapter instanceof FulltextSearchableInterface
&& !($adapter instanceof ValueAnnotationAdapter)) {
&& !($adapter instanceof ResourceAdapter)
&& !($adapter instanceof ValueAnnotationAdapter)
) {
$page = 1;
do {
if ($this->shouldStop()) {
Expand Down
2 changes: 1 addition & 1 deletion application/src/Service/FulltextSearchFactory.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ class FulltextSearchFactory implements FactoryInterface
{
public function __invoke(ContainerInterface $services, $requestedName, array $options = null)
{
return new FulltextSearch($services->get('Omeka\EntityManager'));
return new FulltextSearch($services->get('Omeka\Connection'));
}
}
60 changes: 32 additions & 28 deletions application/src/Stdlib/FulltextSearch.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@
use Omeka\Api\Adapter\AdapterInterface;
use Omeka\Api\Adapter\FulltextSearchableInterface;
use Omeka\Api\ResourceInterface;
use Omeka\Entity\FulltextSearch as FulltextSearchEntity;
use PDO;

class FulltextSearch
{
protected $em;
protected $conn;

public function __construct($em)
public function __construct($conn)
{
$this->em = $em;
$this->conn = $conn;
}

/**
Expand All @@ -26,21 +26,26 @@ public function save(ResourceInterface $resource, AdapterInterface $adapter)
if (!($adapter instanceof FulltextSearchableInterface)) {
return;
}
$searchId = $resource->getId();
$searchResource = $adapter->getResourceName();
$search = $this->em->find(
'Omeka\Entity\FulltextSearch',
['id' => $searchId, 'resource' => $searchResource]
);
if (!$search) {
$search = new FulltextSearchEntity($searchId, $searchResource);
$this->em->persist($search);
}
$search->setOwner($adapter->getFulltextOwner($resource));
$search->setIsPublic($adapter->getFulltextIsPublic($resource));
$search->setTitle($adapter->getFulltextTitle($resource));
$search->setText($adapter->getFulltextText($resource));
$this->em->flush($search);
$resourceId = $resource->getId();
$resourceName = $adapter->getResourceName();
$owner = $adapter->getFulltextOwner($resource);
$ownerId = $owner ? $owner->getId() : null;

$sql = 'INSERT INTO `fulltext_search` (
`id`, `resource`, `owner_id`, `is_public`, `title`, `text`
) VALUES (
:id, :resource, :owner_id, :is_public, :title, :text
) ON DUPLICATE KEY UPDATE
`owner_id` = :owner_id, `is_public` = :is_public, `title` = :title, `text` = :text';
$stmt = $this->conn->prepare($sql);

$stmt->bindValue('id', $resourceId, PDO::PARAM_INT);
$stmt->bindValue('resource', $resourceName, PDO::PARAM_STR);
$stmt->bindValue('owner_id', $ownerId, PDO::PARAM_INT);
$stmt->bindValue('is_public', $adapter->getFulltextIsPublic($resource), PDO::PARAM_BOOL);
$stmt->bindValue('title', $adapter->getFulltextTitle($resource), PDO::PARAM_STR);
$stmt->bindValue('text', $adapter->getFulltextText($resource), PDO::PARAM_STR);
$stmt->executeStatement();
}

/**
Expand All @@ -54,14 +59,13 @@ public function delete($resourceId, AdapterInterface $adapter)
if (!($adapter instanceof FulltextSearchableInterface)) {
return;
}
$searchResource = $adapter->getResourceName();
$search = $this->em->find(
'Omeka\Entity\FulltextSearch',
['id' => $resourceId, 'resource' => $searchResource]
);
if ($search) {
$this->em->remove($search);
$this->em->flush($search);
}
$resourceName = $adapter->getResourceName();

$sql = 'DELETE FROM `fulltext_search` WHERE `id` = :id AND `resource` = :resource';
$stmt = $this->conn->prepare($sql);

$stmt->bindValue('id', $resourceId, PDO::PARAM_INT);
$stmt->bindValue('resource', $resourceName, PDO::PARAM_STR);
$stmt->executeStatement();
}
}

0 comments on commit 6d1cae1

Please sign in to comment.