From 59ed4a4089d90263612334daa7e7b5d2575864d7 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 4 Oct 2023 11:35:28 -0100 Subject: [PATCH] FilesMetadata Signed-off-by: Maxence Lange --- core/Command/FilesMetadata/Get.php | 44 +++++ .../Version28000Date20231004103301.php | 73 ++++++++ core/register_command.php | 2 + .../Event/FilesMetadataEvent.php | 57 ++++++ .../FilesMetadata/FilesMetadataManager.php | 170 ++++++++++++++++++ .../FilesMetadataQueryHelper.php | 53 ++++++ .../FilesMetadata/Model/FilesMetadata.php | 125 +++++++++++++ lib/private/Server.php | 3 + .../FilesMetadataNotFoundException.php | 10 ++ lib/public/FilesMetadata/IFilesMetadata.php | 58 ++++++ .../FilesMetadata/IFilesMetadataManager.php | 19 ++ .../IFilesMetadataQueryHelper.php | 31 ++++ 12 files changed, 645 insertions(+) create mode 100644 core/Command/FilesMetadata/Get.php create mode 100644 core/Migrations/Version28000Date20231004103301.php create mode 100644 lib/private/FilesMetadata/Event/FilesMetadataEvent.php create mode 100644 lib/private/FilesMetadata/FilesMetadataManager.php create mode 100644 lib/private/FilesMetadata/FilesMetadataQueryHelper.php create mode 100644 lib/private/FilesMetadata/Model/FilesMetadata.php create mode 100644 lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php create mode 100644 lib/public/FilesMetadata/IFilesMetadata.php create mode 100644 lib/public/FilesMetadata/IFilesMetadataManager.php create mode 100644 lib/public/FilesMetadata/IFilesMetadataQueryHelper.php diff --git a/core/Command/FilesMetadata/Get.php b/core/Command/FilesMetadata/Get.php new file mode 100644 index 0000000000000..99c1dd74b341b --- /dev/null +++ b/core/Command/FilesMetadata/Get.php @@ -0,0 +1,44 @@ +setName('metadata:get') + ->setDescription('update and returns up-to-date metadata') + ->addArgument( + 'fileId', + InputArgument::REQUIRED, + 'id of the file document' + ) + ->addOption( + 'background', + '', + InputOption::VALUE_NONE, + 'emulate background jobs env' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $fileId = (int) $input->getArgument('fileId'); + $metadata = $this->filesMetadataManager->refreshMetadata($fileId); + $output->writeln(json_encode($metadata, JSON_PRETTY_PRINT)); + + return 0; + } +} diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php new file mode 100644 index 0000000000000..b11358cfbbc93 --- /dev/null +++ b/core/Migrations/Version28000Date20231004103301.php @@ -0,0 +1,73 @@ +hasTable('files_metadata')) { + $table = $schema->createTable('files_metadata'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 15, + ]); + $table->addColumn('json', Types::TEXT); + $table->addColumn('sync_token', Types::STRING, [ + 'length' => 15, + ]); + $table->addColumn('last_update', Types::DATETIME); + + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id'], 'files_meta_fileid'); + } + + if (!$schema->hasTable('files_metadata_index')) { + $table = $schema->createTable('files_metadata_index'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 15, + ]); + $table->addColumn('k', Types::STRING, [ + 'notnull' => false, + 'length' => 31, + ]); + $table->addColumn('v', Types::STRING, [ + 'notnull' => false, + 'length' => 63, + ]); + $table->addColumn('v_int', Types::BIGINT, [ + 'notnull' => false, + 'length' => 11, + ]); + $table->addColumn('last_update', Types::DATETIME); + + $table->setPrimaryKey(['id']); + $table->addIndex(['k', 'v', 'v_int'], 'files_meta_indexes'); + } + + return $schema; + } +} diff --git a/core/register_command.php b/core/register_command.php index d9e5dfcd775eb..d497a9581d1af 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -214,6 +214,8 @@ $application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager())); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class)); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class)); + + $application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class)); } else { $application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class)); } diff --git a/lib/private/FilesMetadata/Event/FilesMetadataEvent.php b/lib/private/FilesMetadata/Event/FilesMetadataEvent.php new file mode 100644 index 0000000000000..9a56efacde8b3 --- /dev/null +++ b/lib/private/FilesMetadata/Event/FilesMetadataEvent.php @@ -0,0 +1,57 @@ +runAsBackgroundJob = true; + } + + /** + * return fileId + * + * @return int + */ + public function getFileId(): int { + return $this->fileId; + } + + /** + * return Metadata + * + * @return IFilesMetadata + */ + public function getMetadata(): IFilesMetadata { + return $this->metadata; + } + + /** + * return true if any app that catch this event requested a re-run as background job + * + * @return bool + */ + public function isRunAsBackgroundJobRequested(): bool { + return $this->runAsBackgroundJob; + } +} diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php new file mode 100644 index 0000000000000..409ac9b24d93e --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -0,0 +1,170 @@ +selectMetadata($fileId); + } catch (FilesMetadataNotFoundException $e) { + } + } + + if (is_null($metadata)) { + $metadata = new FilesMetadata($fileId); + } + + $event = new FilesMetadataEvent($fileId, $metadata); + $this->eventDispatcher->dispatchTyped($event); + $this->saveMetadata($event->getMetadata()); + + return $metadata; + } + + /** + * @param int $fileId + * @param bool $createIfNeeded + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + public function getMetadata(int $fileId, bool $createIfNeeded = false): IFilesMetadata { + try { + return $this->selectMetadata($fileId); + } catch (FilesMetadataNotFoundException $e) { + if ($createIfNeeded) { + return $this->refreshMetadata($fileId); + } + + throw $e; + } + } + + + public function saveMetadata(IFilesMetadata $filesMetadata): void { + if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) { + return; + } + + try { + try { + $this->insertMetadata($filesMetadata); + } catch (UniqueConstraintViolationException $e) { + $this->updateMetadata($filesMetadata); + } + } catch (Exception $e) { + $this->logger->warning('exception while saveMetadata()', ['exception' => $e]); + + return; + } + +// $this->removeDeprecatedMetadata($filesMetadata); + // remove indexes from metadata_index that are not in the list of indexes anymore. + foreach ($filesMetadata->listIndexes() as $index) { + // foreach index, update entry in table metadata_index + // if no update, insert as new row + // !! we might want to get the type of the value to be indexed at one point !! + } + } + + private function removeDeprecatedMetadata(IFilesMetadata $filesMetadata): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->notIn('file_id', $filesMetadata->listIndexes(), IQueryBuilder::PARAM_STR_ARRAY)); + $qb->executeStatement(); + } + + + public function getQueryHelper(): IFilesMetadataQueryHelper { + return $this->filesMetadataQueryHelper; + } + + private function insertMetadata(IFilesMetadata $filesMetadata): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA) + ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) + ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->setValue('sync_token', $qb->createNamedParameter('abc')) + ->setValue('last_update', $qb->createFunction('NOW()')); + $qb->execute(); + } + + /** + * @param IFilesMetadata $filesMetadata + * + * @return bool + * @throws Exception + */ + private function updateMetadata(IFilesMetadata $filesMetadata): void { + // TODO check sync_token on update + $qb = $this->dbConnection->getQueryBuilder(); + $qb->update(self::TABLE_METADATA) + ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->set('sync_token', $qb->createNamedParameter('abc')) + ->set('last_update', $qb->createFunction('NOW()')) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + } + + /** + * @param int $fileId + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + private function selectMetadata(int $fileId): IFilesMetadata { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('json')->from(self::TABLE_METADATA); + $qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + } catch (Exception $e) { + $this->logger->warning('exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]); + throw new FilesMetadataNotFoundException(); + } + + if ($data === false) { + throw new FilesMetadataNotFoundException(); + } + + $metadata = new FilesMetadata($fileId); + $metadata->import($data['json'] ?? ''); + + return $metadata; + } +} diff --git a/lib/private/FilesMetadata/FilesMetadataQueryHelper.php b/lib/private/FilesMetadata/FilesMetadataQueryHelper.php new file mode 100644 index 0000000000000..cde5a117be61d --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataQueryHelper.php @@ -0,0 +1,53 @@ +import($json); + + return $metadata; + } + +} diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php new file mode 100644 index 0000000000000..38cbc00d7afbb --- /dev/null +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -0,0 +1,125 @@ +fileId = $fileId; + $this->metadata = $metadata; + } + + public function getFileId(): int { + return $this->fileId; + } + + public function import(string $json): IFilesMetadata { + $this->metadata = json_decode($json, true, JSON_THROW_ON_ERROR); + $this->updated = false; + return $this; + } + + public function updated(): bool { + return $this->updated; + } + + public function lastUpdateTimestamp(): int { + return $this->lastUpdate; + } + public function getSyncToken(): string { + return $this->syncToken; + } + public function hasKey(string $needle): bool { + return (in_array($needle, $this->getKeys())); + } + public function getKeys(): array { + return array_diff(array_keys($this->metadata, [self::INDEXES_KEY])); + } + + public function listIndexes(): array { + return $this->getArray(self::INDEXES_KEY, []); + } + + public function addIndex(string $index): IFilesMetadata { + if (!array_key_exists('_indexes', $this->metadata)) { + $this->metadata[self::INDEXES_KEY] = []; + } + + $this->metadata[self::INDEXES_KEY][] = $index; + return $this; + } + + public function removeIndex(string $index): IFilesMetadata { + if (!array_key_exists(self::INDEXES_KEY, $this->metadata)) { + return $this; + } + + $this->metadata[self::INDEXES_KEY] = array_diff($this->metadata[self::INDEXES_KEY], [$index]); + return $this; + } + + public function get(string $key, string $default): string { + return $this->metadata[$key] ?? $default; + } + + public function getInt(string $key, int $default): int { + return $this->metadata[$key] ?? $default; + } + + public function getFloat(string $key, float $default): float { + return $this->metadata[$key] ?? $default; + } + + public function getBool(string $key, bool $default): bool { + return $this->metadata[$key] ?? $default; + } + + public function getArray(string $key, array $default): array { + return $this->metadata[$key] ?? $default; + } + + public function set(string $key, string $value): IFilesMetadata { + $this->metadata[$key] = $value; + $this->updated = true; + return $this; + } + + public function setInt(string $key, int $value): IFilesMetadata { + $this->metadata[$key] = $value; + $this->updated = true; + return $this; + } + + public function setFloat(string $key, float $value): IFilesMetadata { + $this->metadata[$key] = $value; + $this->updated = true; + return $this; + } + + public function setBool(string $key, bool $value): IFilesMetadata { + $this->metadata[$key] = $value; + $this->updated = true; + return $this; + } + + public function setArray(string $key, array $value): IFilesMetadata { + $this->metadata[$key] = $value; + $this->updated = true; + return $this; + } + + public function jsonSerialize() { + return $this->metadata; + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index d2a1d890ccd25..34a9de845757c 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -102,6 +102,7 @@ use OC\Files\Template\TemplateManager; use OC\Files\Type\Loader; use OC\Files\View; +use OC\FilesMetadata\FilesMetadataManager; use OC\FullTextSearch\FullTextSearchManager; use OC\Http\Client\ClientService; use OC\Http\Client\NegativeDnsCache; @@ -193,6 +194,7 @@ use OCP\Files\Mount\IMountManager; use OCP\Files\Storage\IStorageFactory; use OCP\Files\Template\ITemplateManager; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FullTextSearch\IFullTextSearchManager; use OCP\GlobalScale\IConfig; use OCP\Group\ISubAdmin; @@ -1395,6 +1397,7 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(\OCP\Dashboard\IManager::class, \OC\Dashboard\Manager::class); $this->registerAlias(IFullTextSearchManager::class, FullTextSearchManager::class); + $this->registerAlias(IFilesMetadataManager::class, FilesMetadataManager::class); $this->registerAlias(ISubAdmin::class, SubAdmin::class); diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php new file mode 100644 index 0000000000000..a5f14e3622885 --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php @@ -0,0 +1,10 @@ +