From 0984bfed3d3b72c98570ba5a74a185814ad59419 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Mon, 16 Mar 2020 21:08:12 -0400 Subject: [PATCH 1/6] First pass on SBF runners plugin system This is a playground, not ready for propagation of strawberries for mass consumption yet. --- LICENSE | 165 +++++++++++ README.md | 34 +++ composer.json | 26 ++ config/schema/strawberry_runners.schema.yml | 56 ++++ js/jmespath-codemirror_strawberry_runners.js | 16 + src/Ajax/UpdateCodeMirrorCommand.php | 53 ++++ .../StrawberryRunnersPostProcessor.php | 54 ++++ .../StrawberryRunnersToolsController.php | 177 +++++++++++ .../StrawberryRunnersWebhookController.php | 123 ++++++++ ...ryRunnerPostProcessorEntityListBuilder.php | 121 ++++++++ .../strawberryRunnerPostprocessorEntity.php | 166 +++++++++++ ...erryRunnerPostprocessorEntityInterface.php | 32 ++ ...ryRunnersEventJsonProcessingSubscriber.php | 280 ++++++++++++++++++ src/Form/AdoFileDeleteForm.php | 129 ++++++++ src/Form/StrawberryRunnersToolsForm.php | 180 +++++++++++ ...rryRunnerPostprocessorEntityDeleteForm.php | 56 ++++ ...trawberryRunnerPostprocessorEntityForm.php | 167 +++++++++++ .../ProcessWebhookPayloadQueueWorker.php | 140 +++++++++ .../SystemBinaryPostProcessor.php | 226 ++++++++++++++ ...rawberryRunnersPostProcessorPluginBase.php | 136 +++++++++ ...rryRunnersPostProcessorPluginInterface.php | 72 +++++ ...berryRunnersPostProcessorPluginManager.php | 43 +++ ...erPostProcessorEntityHtmlRouteProvider.php | 25 ++ strawberry_runners.info.yml | 10 + strawberry_runners.install | 32 ++ strawberry_runners.libraries.yml | 7 + strawberry_runners.links.action.yml | 6 + strawberry_runners.links.menu.yml | 7 + strawberry_runners.links.task.yml | 4 + strawberry_runners.module | 11 + strawberry_runners.routing.yml | 30 ++ strawberry_runners.services.yml | 4 + 32 files changed, 2588 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 composer.json create mode 100644 config/schema/strawberry_runners.schema.yml create mode 100644 js/jmespath-codemirror_strawberry_runners.js create mode 100644 src/Ajax/UpdateCodeMirrorCommand.php create mode 100644 src/Annotation/StrawberryRunnersPostProcessor.php create mode 100644 src/Controller/StrawberryRunnersToolsController.php create mode 100644 src/Controller/StrawberryRunnersWebhookController.php create mode 100644 src/Entity/Controller/strawberryRunnerPostProcessorEntityListBuilder.php create mode 100644 src/Entity/strawberryRunnerPostprocessorEntity.php create mode 100644 src/Entity/strawberryRunnerPostprocessorEntityInterface.php create mode 100644 src/EventSubscriber/StrawberryRunnersEventJsonProcessingSubscriber.php create mode 100644 src/Form/AdoFileDeleteForm.php create mode 100644 src/Form/StrawberryRunnersToolsForm.php create mode 100644 src/Form/strawberryRunnerPostprocessorEntityDeleteForm.php create mode 100644 src/Form/strawberryRunnerPostprocessorEntityForm.php create mode 100644 src/Plugin/QueueWorker/ProcessWebhookPayloadQueueWorker.php create mode 100644 src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php create mode 100644 src/Plugin/StrawberryRunnersPostProcessorPluginBase.php create mode 100644 src/Plugin/StrawberryRunnersPostProcessorPluginInterface.php create mode 100644 src/Plugin/StrawberryRunnersPostProcessorPluginManager.php create mode 100644 src/strawberryRunnerPostProcessorEntityHtmlRouteProvider.php create mode 100644 strawberry_runners.info.yml create mode 100644 strawberry_runners.install create mode 100644 strawberry_runners.libraries.yml create mode 100644 strawberry_runners.links.action.yml create mode 100644 strawberry_runners.links.menu.yml create mode 100644 strawberry_runners.links.task.yml create mode 100644 strawberry_runners.module create mode 100644 strawberry_runners.routing.yml create mode 100644 strawberry_runners.services.yml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..65c5ca8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ff26546 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# Disclaimer +This is an experimental module, use at your own risk + +# Strawberry Runners + +A Drupal 8 module that provides a set of post processing capabilities for JSON based metadata, files and entities, based on Dispatched +events, direct http calls and invoked webhooks from partner services (e.g Min.io, AWS S3 or self invoked). This is part of the Archipelago Commons Project. + +This module adds to each D8/Archipelago instance headless processing services (something like embeded services instead of microservice) that can be invoked +internally or externally to make use of multiple concurrent HTTP services via reactphp. + +## Help + +Having issues with this module? Check out the Archipelago Commons google groups for tech + emotional support + updates. + +* [Archipelago Commons](https://groups.google.com/forum/#!forum/archipelago-commons) + +## Demo + +* archipelago.nyc (http://archipelago.nyc) + +## Caring & Coding + Fixing + +* [Giancarlo Birello](https://github.com/giancarlobi) +* [Diego Pino](https://github.com/DiegoPino) + +## Acknowledgments + +This software is a [Metropolitan New York Library Council](https://metro.org) Open-Source initiative and part of the Archipelago Commons project. + +## License + +[GPLv3](http://www.gnu.org/licenses/gpl-3.0.txt) + diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..e80b70f --- /dev/null +++ b/composer.json @@ -0,0 +1,26 @@ +{ + "name": "strawberryfield/strawberry_runners", + "description": "Strawberryfield post processing module for Drupal 8", + "type": "drupal-module", + "license": "GPL-2.0+", + "homepage": "https://github.com/esmero/strawberry_runners", + "authors": [ + { + "name": "Giancarlo Birello", + "email": "giancarlo.birello@gmail.com", + "homepage": "https://github.com/giancarlobi", + "role": "Owner" + }, + { + "name": "Diego Pino Navarro", + "email": "dpino@metro.org", + "homepage": "https://github.com/DiegoPino", + "role": "Owner" + } + ], + "require": { + "ml/json-ld": "^1.0", + "mtdowling/jmespath.php":"^2.4", + "strawberryfield/strawberryfield":"dev-8.x-1.0-beta2" + } +} diff --git a/config/schema/strawberry_runners.schema.yml b/config/schema/strawberry_runners.schema.yml new file mode 100644 index 0000000..4fda04d --- /dev/null +++ b/config/schema/strawberry_runners.schema.yml @@ -0,0 +1,56 @@ +strawberryfield_runners.strawberry_runners_postprocessor.*: + type: config_entity + label: 'Strawberry Runners Post Processor Config Entity' + mapping: + id: + type: string + label: 'ID' + label: + type: label + label: 'Label' + uuid: + type: string + pluginid: + type: string + label: 'Plugin ID' + pluginconfig: + type: strawberryfield_runners.strawberry_runners_postprocessor.[%parent.pluginid] + active: + type: boolean + label: 'Whether this plugin is active or not' + +strawberryfield_runners.strawberry_runners_postprocessor.binary: + type: config_object + label: 'Strawberry Runners Post Processor Config Entity Binary specific config' + mapping: + source_type: + type: string + label: 'The type of Source Data this Processor works on' + ado_type: + type: string + label: 'DO type(s) to limit this Processor to' + jsonkey: + type: sequence + label: 'The JSON key(s) containing the desired Source File(s)' + sequence: + - type: string + mime_type: + type: string + label: 'Mimetypes(s) to limit this Processor to' + arguments: + type: string + label: 'Any additional argument your executable binary requires' + output_type: + type: string + label: 'The expected and desired output of this processor' + output_destination: + type: sequence + label: 'Where and how the output will be used' + sequence: + - type: string + timeout: + type: integer + label: 'Timeout in seconds for this process' + weight: + type: integer + label: 'Order or execution in the global chain' \ No newline at end of file diff --git a/js/jmespath-codemirror_strawberry_runners.js b/js/jmespath-codemirror_strawberry_runners.js new file mode 100644 index 0000000..4e507d6 --- /dev/null +++ b/js/jmespath-codemirror_strawberry_runners.js @@ -0,0 +1,16 @@ +(function ($, Drupal) { + Drupal.AjaxCommands.prototype.strawberry_runners_codemirror = function (ajax, response, status) { + if (!window.CodeMirror) { + return; + } + + $editors = $(response.selector).find('.CodeMirror'); + + if (response.hasOwnProperty('content') && + $editors.length > 0 ) { + console.log('we have content'); + $editors[0].CodeMirror.setValue(response.content); + } + }; + +})(jQuery, Drupal); \ No newline at end of file diff --git a/src/Ajax/UpdateCodeMirrorCommand.php b/src/Ajax/UpdateCodeMirrorCommand.php new file mode 100644 index 0000000..ba8b82e --- /dev/null +++ b/src/Ajax/UpdateCodeMirrorCommand.php @@ -0,0 +1,53 @@ +selector = $selector; + $this->content = $content; + + } + + /** + * Implements Drupal\Core\Ajax\CommandInterface:render(). + */ + public function render() { + + return [ + 'command' => 'strawberry_runners_codemirror', + 'selector' => $this->selector, + 'content' => $this->content, + ]; + } + +} + diff --git a/src/Annotation/StrawberryRunnersPostProcessor.php b/src/Annotation/StrawberryRunnersPostProcessor.php new file mode 100644 index 0000000..843d5aa --- /dev/null +++ b/src/Annotation/StrawberryRunnersPostProcessor.php @@ -0,0 +1,54 @@ +renderer = $renderer; + $this->entityRepository = $entity_repository; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('renderer'), + $container->get('entity.repository') + ); + } + + + /** + * Generates an overview table of all Tools. + * + * @param \Drupal\node\NodeInterface $node + * A node object. + * + * @return array + * An array as expected by \Drupal\Core\Render\RendererInterface::render(). + */ + public function toolsOverview(NodeInterface $node) { + + + $build['test_jmespath'] = [ + '#name' => 'jmespath_tester', + '#type' => 'textfield', + '#title' => $this->t('Email address'), + '#ajax' => [ + 'callback' => [$this, 'callJmesPathprocess'], + 'event' => 'change', + 'keypress' => TRUE, + 'disable-refocus' => TRUE + ] + ]; + + /* + $account = $this->currentUser(); + $node_storage = $this->entityTypeManager()->getStorage('node'); + $type = $node->getType(); + + $build['#title'] = $this->t('Associated Media for %title', ['%title' => $node->label()]); + $header = [$this->t('Revision'), $this->t('Operations')]; + $delete_permission = (($account->hasPermission("delete $type revisions") || $account->hasPermission('delete all revisions') || $account->hasPermission('administer nodes')) && $node->access('delete')); + + $rows = []; + + + foreach ($this->getRevisionIds($node, $node_storage) as $vid) { + $revision = $node_storage->loadRevision($vid); + + $row = []; + $column = [ + 'data' => [ + '#type' => 'inline_template', + '#template' => '{% trans %}{{ date }} by {{ username }}{% endtrans %}{% if message %}

{{ message }}

{% endif %}', + '#context' => [ + 'date' => 'blabal', + 'username' => $this->renderer->renderPlain($username), + 'message' => ['#markup' => $revision->revision_log->value, '#allowed_tags' => Xss::getHtmlTagList()], + ], + ], + ]; + $this->renderer->addCacheableDependency($column['data'], $username); + $row[] = $column; + + + $links = []; + + if ($delete_permission) { + $links['delete'] = [ + 'title' => $this->t('Delete'), + 'url' => Url::fromRoute('node.revision_delete_confirm', ['node' => $node->id(), 'node_revision' => $vid]), + ]; + } + + $row[] = [ + 'data' => [ + '#type' => 'operations', + '#links' => $links, + ], + ]; + + $rows[] = $row; + } + + + $build['file_table'] = [ + '#theme' => 'table', + '#rows' => $rows, + '#header' => $header, + ]; + + $build['pager'] = ['#type' => 'pager']; + */ + + return $build; + } + + /** + * Limit access to the Tools according to their restricted state. + * + * @param \Drupal\Core\Session\AccountInterface $account + * The account object. + * @param int $node + * The node id. + */ + public function accessTools(AccountInterface $account, $node) { + $node_storage = $this->entityTypeManager()->getStorage('node'); + $node = $node_storage->load($node); + $type = $node->getType(); + // @TODO for now... + if ($sbf_fields = \Drupal::service('strawberryfield.utility')->bearsStrawberryfield($node)) { + $access = AccessResult::allowedIfHasPermission($account, 'edit any ' . $type . ' content'); + if (!$access->isAllowed() && $account->hasPermission('edit own ' . $type . ' content')) { + $access = $access->orIf(AccessResult::allowedIf($account->id() == $node->getOwnerId())->cachePerUser()->addCacheableDependency($node)); + } + } else { + $access = AccessResult::forbidden(); + } + + $access->addCacheableDependency($node); + return AccessResult::allowedIf($access)->cachePerPermissions(); + + + } + +} diff --git a/src/Controller/StrawberryRunnersWebhookController.php b/src/Controller/StrawberryRunnersWebhookController.php new file mode 100644 index 0000000..441aa73 --- /dev/null +++ b/src/Controller/StrawberryRunnersWebhookController.php @@ -0,0 +1,123 @@ +logger = $logger->get('strawberry_runners'); + $this->queue = $queue; + $secret = \Drupal::service('config.factory')->get('strawberry_runners')->get('webhooktoken'); + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('logger.factory'), + $container->get('queue')->get('process_payload_queue_worker') + ); + } + + /** + * Capture the payload. + * + * @return \Symfony\Component\HttpFoundation\Response + * A simple string and 200 response. + */ + public function capture(Request $request) { + + $response = new Response(); + + // Capture the payload. + $payload = $request->getContent(); + + // Check if it is empty. + if (empty($payload)) { + $message = 'The Webhook payload was empty.'; + $this->logger->error($message); + $response->setContent($message); + return $response; + } + + // Use temporarily to inspect payload. + if ($this->debug) { + $this->logger->debug('
@payload
', ['@payload' => $payload]); + } + + // Add the $payload to our defined queue. + $this->queue->createItem($payload); + + $response->setContent('Success!'); + return $response; + } + + /** + * Simple authorization using a token. + * + * @param string $token + * A random token only your webhook knows about. + * + * @return AccessResult + * AccessResult allowed or forbidden. + */ + public function authorize($token) { + // Always require a token. + if (($token === $this->secret) && !empty($token)) { + return AccessResult::allowed(); + } + return AccessResult::forbidden(); + } + +} \ No newline at end of file diff --git a/src/Entity/Controller/strawberryRunnerPostProcessorEntityListBuilder.php b/src/Entity/Controller/strawberryRunnerPostProcessorEntityListBuilder.php new file mode 100644 index 0000000..71d80ab --- /dev/null +++ b/src/Entity/Controller/strawberryRunnerPostProcessorEntityListBuilder.php @@ -0,0 +1,121 @@ +configFactory = $config_factory; + $this->messenger = $messenger; + } + + /** + * {@inheritdoc} + */ + public static function createInstance(ContainerInterface $container, EntityTypeInterface $entity_type) { + return new static( + $entity_type, + $container->get('entity_type.manager')->getStorage($entity_type->id()), + $container->get('config.factory'), + $container->get('messenger') + ); + } + /** + * {@inheritdoc} + * + * We override ::render() so that we can add our own content above the table. + * parent::render() is where EntityListBuilder creates the table using our + * buildHeader() and buildRow() implementations. + */ + public function render() { + $build['description'] = [ + '#markup' => $this->t( + 'Strawberry Runners Module implements Post processor Plugins that enhance Metadata or do fun things with Files present in each Node that contains a Strawberryfield type of field (ADO).' + ), + ]; + + $build += parent::render(); + return $build; + } + + /** + * {@inheritdoc} + * + * Building the header and content lines for the SBR list. + * + * Calling the parent::buildHeader() adds a column for the possible actions + * and inserts the 'edit' and 'delete' links as defined for the entity type. + */ + public function buildHeader() { + $header['id'] = $this->t('Post Processor Config ID'); + $header['label'] = $this->t('Label'); + $header['active'] = $this->t('Is active ?'); + return $header + parent::buildHeader(); + } + + /** + * {@inheritdoc} + */ + public function buildRow(EntityInterface $entity) { + /* @var $entity \Drupal\strawberry_runners\Entity\strawberryRunnerPostprocessorEntity */ + $row['id'] = [ '#markup' => $entity->id() ]; + $row['label'] = $entity->label(); + $row['active'] = $entity->isActive() ? [ '#markup' => $this->t('Yes')] : [ '#markup' =>$this->t('No')]; + + return $row + parent::buildRow($entity); + } + + +} diff --git a/src/Entity/strawberryRunnerPostprocessorEntity.php b/src/Entity/strawberryRunnerPostprocessorEntity.php new file mode 100644 index 0000000..1089503 --- /dev/null +++ b/src/Entity/strawberryRunnerPostprocessorEntity.php @@ -0,0 +1,166 @@ +pluginid ?: ''; + } + + /** + * @param string $pluginid + */ + public function setPluginid(string $pluginid): void { + $this->pluginid = $pluginid; + } + + /** + * @return bool + */ + public function isActive(): bool { + return $this->active; + } + + /** + * @param bool $active + */ + public function setActive(bool $active): void { + $this->active = $active; + } + + /** + * @return array + */ + public function getPluginconfig(): array { + return $this->pluginconfig ?:[]; + } + + /** + * @param array $pluginconfig + */ + public function setPluginconfig(array $pluginconfig): void { + $this->pluginconfig = $pluginconfig; + } + + + /** + * Sorts the Post processor entities, putting disabled ones at the bottom. + * + * @see \Drupal\Core\Config\Entity\ConfigEntityBase::sort() + */ + public static function sort(ConfigEntityInterface $a, ConfigEntityInterface $b) { + + // Check if the entities are flags, if not go with the default. + if ($a instanceof strawberryRunnerPostprocessorEntityInterface && $b instanceof strawberryRunnerPostprocessorEntityInterface) { + + if ($a->isActive() && $b->isActive()) { + return parent::sort($a, $b); + } + elseif (!$a->isActive()) { + return -1; + } + elseif (!$b->isActive()) { + return 1; + } + } + + return parent::sort($a, $b); + } + +} diff --git a/src/Entity/strawberryRunnerPostprocessorEntityInterface.php b/src/Entity/strawberryRunnerPostprocessorEntityInterface.php new file mode 100644 index 0000000..def6aa2 --- /dev/null +++ b/src/Entity/strawberryRunnerPostprocessorEntityInterface.php @@ -0,0 +1,32 @@ +stringTranslation = $string_translation; + $this->messenger = $messenger; + $this->loggerFactory = $logger_factory; + $this->configFactory = $config_factory; + $this->storage = $storage; + $this->entityTypeManager = $entity_type_manager; + $this->streamWrapperManager = $stream_wrapper_manager; + $this->fileSystem = $file_system; + $this->strawberryRunnerProcessorPluginManager = $strawberry_runner_processor_plugin_manager; + } + + /** + * Method called when Event occurs. + * + * @param \Drupal\strawberryfield\Event\StrawberryfieldServiceEvent $event + * The event. + */ + public function onJsonInvokeProcess(StrawberryfieldServiceEvent $event) { + + /* @var $plugin_config_entities \Drupal\strawberry_runners\Entity\strawberryRunnerPostprocessorEntity[] */ + $plugin_config_entities = $this->entityTypeManager->getListBuilder('strawberry_runners_postprocessor')->load(); + $active_plugins = []; + foreach($plugin_config_entities as $plugin_config_entity) { + if ($plugin_config_entity->isActive()) { + $entity_id = $plugin_config_entity->id(); + $configuration_options = $plugin_config_entity->getPluginconfig(); + $configuration_options['configEntity'] = $entity_id; + /* @var \Drupal\strawberry_runners\Plugin\StrawberryRunnersPostProcessorPluginInterface $plugin_instance */ + $plugin_instance = $this->strawberryRunnerProcessorPluginManager->createInstance( + $plugin_config_entity->getPluginid(), + $configuration_options + ); + $plugin_definition = $plugin_instance->getPluginDefinition(); + // We don't use the key here to preserve the original weight given order + // Classify by input type + $active_plugins[$plugin_definition['input_type']][] = $plugin_instance; + } + } + + // We will fetch all files and then see if each file can be processed by one + // or more plugin. + // Slower option would be to traverse every file per processor. + + $updated = 0; + $entity = $event->getEntity(); + $sbf_fields = $event->getFields(); + $processedcount = 0; + foreach ($sbf_fields as $field_name) { + /* @var $field \Drupal\Core\Field\FieldItemInterface */ + $field = $entity->get($field_name); + if (!$field->isEmpty()) { + $entity = $field->getEntity(); + $entity_type_id = $entity->getEntityTypeId(); + /** @var $field \Drupal\Core\Field\FieldItemList */ + foreach ($field->getIterator() as $delta => $itemfield) { + // Note: we are not touching the metadata here. + /** @var $itemfield \Drupal\strawberryfield\Plugin\Field\FieldType\StrawberryFieldItem */ + $flatvalues = (array) $itemfield->provideFlatten(); + // Run first on entity:files + + if (isset($flatvalues['dr:fid'])) { + foreach ($flatvalues['dr:fid'] as $fid) { + if (is_numeric($fid)) { + $file = $this->entityTypeManager->getStorage('file')->load( + $fid + ); + /** @var $file FileInterface; */ + if ($file) { + $this->add_file_usage($file, $entity->id(), $entity_type_id); + $updated++; + } + else { + $this->messenger()->addError( + t( + 'Your content references a file with Internal ID @file_id that does not exist or was removed.', + ['@file_id' => $fid] + ) + ); + } + } + } + } + } + } + } + } + + /** + * Move file to local and process. + * + * @param \Drupal\file\FileInterface $file + * The File URI to look at. + * + * @return array + * Output of processing chain for a particular file. + */ + private function processFile(FileInterface $file) { + $uri = $file->getFileUri(); + $processOutput = []; + + /** @var \Drupal\Core\File\FileSystem $file_system */ + $scheme = $this->fileSystem->uriScheme($uri); + + // If the file isn't stored locally make a temporary copy. + if (!isset($this->streamWrapperManager + ->getWrappers(StreamWrapperInterface::LOCAL)[$scheme])) { + // Local stream. + $cache_key = md5($uri); + if (empty($this->instanceCopiesOfFiles[$cache_key])) { + if (!($this->instanceCopiesOfFiles[$cache_key] = $this->fileSystem->copy($uri, 'temporary://sbr_' . $cache_key . '_' . basename($uri), FileSystemInterface::FILE_EXISTS_REPLACE))) { + $this->loggerFactory->get('strawberry_runners') + ->notice('Unable to create local temporary copy of remote file for Strawberry Runners Post processing File %file.', + [ + '%file' => $uri, + ]); + return []; + } + } + $uri = $this->instanceCopiesOfFiles[$cache_key]; + } + + return $processOutput; + } + + /** + * Make sure no HTML or Javascript will be passed around. + * + * @param string $string + * A value returned by a processor + * + * @return string + * The value sanitized. + */ + private function sanitizeValue($string) { + if (!Unicode::validateUtf8($string)) { + $string = Html::escape(utf8_encode($string)); + } + return $string; + } + + /** + * Cleanup of artifacts from processing files. + */ + public function __destruct() { + // Get rid of temporary files created for this instance. + foreach ($this->instanceCopiesOfFiles as $uri) { + \Drupal::service('file_system')->unlink($uri); + } + } + + + + + + + +} diff --git a/src/Form/AdoFileDeleteForm.php b/src/Form/AdoFileDeleteForm.php new file mode 100644 index 0000000..47a6fc8 --- /dev/null +++ b/src/Form/AdoFileDeleteForm.php @@ -0,0 +1,129 @@ +entityTypeManager = $entity_type_manager; + $this->messenger = $messenger; + $this->sbffileservice = $filePersisterService; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('entity.manager'), + $container->get('messenger'), + $container->get('strawberryfield.file_persister') + ); + } + + /** + * {@inheritdoc} + */ + public function getFormId() { + return 'ado_file_delete_form'; + } + + /** + * {@inheritdoc} + */ + public function getQuestion() { + return t('Are you sure you want to remove the file %file attached to this ADO %node', [ + '%node' => $this->node->label(), + '%file' => $this->file->getFilename(), + ]); + } + + /** + * {@inheritdoc} + */ + public function getCancelUrl() { + return new Url('strawberry_runners.ado_tools', ['node' => $this->node->id()]); + } + + /** + * {@inheritdoc} + */ + public function getConfirmText() { + return t('Remove'); + } + + /** + * {@inheritdoc} + */ + public function buildForm(array $form, FormStateInterface $form_state, $node = NULL, $file = NULL) { + $this->node = $this->entityTypeManager->getStorage('node')->load($node); + $this->file = $this->entityTypeManager->getStorage('file')->load($file); + + $form = parent::buildForm($form, $form_state); + + return $form; + } + + /** + * {@inheritdoc} + */ + public function submitForm(array &$form, FormStateInterface $form_state) { + + $this->logger('content')->notice('@type: deleted %title revision %revision.', ['@type' => $this->revision->bundle(), '%title' => $this->revision->label(), '%revision' => $this->revision->getRevisionId()]); + $this->messenger->addStatus('Done!'); + $form_state->setRedirect( + 'strawberry_runners.ado_tools', + ['node' => $this->node->id()] + ); + } + +} diff --git a/src/Form/StrawberryRunnersToolsForm.php b/src/Form/StrawberryRunnersToolsForm.php new file mode 100644 index 0000000..c296b65 --- /dev/null +++ b/src/Form/StrawberryRunnersToolsForm.php @@ -0,0 +1,180 @@ +entityRepository = $entity_repository; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('renderer'), + $container->get('entity.repository') + ); + } + + public function getFormId() { + return 'strawberry_runners_tools_form'; + } + + public function buildForm(array $form, FormStateInterface $form_state, NodeInterface $node = NULL) { + + // For code Mirror + // @TODO make this module dependant + $settings['mode'] = 'application/ld+json'; + $settings['readOnly'] = TRUE; + $settings['toolbar'] = FALSE; + $settings['lineNumbers'] = TRUE; + error_log('ups i created a form'); + if ($sbf_fields = \Drupal::service('strawberryfield.utility')->bearsStrawberryfield($node)) { + foreach ($sbf_fields as $field_name) { + /* @var $field \Drupal\Core\Field\FieldItemInterface */ + $field = $node->get($field_name); + if (!$field->isEmpty()) { + /** @var $field \Drupal\Core\Field\FieldItemList */ + foreach ($field->getIterator() as $delta => $itemfield) { + // Note: we are not longer touching the metadata here. + /** @var $itemfield \Drupal\strawberryfield\Plugin\Field\FieldType\StrawberryFieldItem */ + $json = json_encode(json_decode($itemfield->value), JSON_PRETTY_PRINT); + $form_state->set('itemfield', $itemfield); + $form['test_jmespath'] = [ + '#type' => 'textfield', + '#default_value' => $form_state->getValue('test_jmespath'), + '#title' => $this->t('JMESPATH'), + '#description' => $this->t( + 'Evaluate a JMESPath Query against this ADO\'s JSON. See JMESPath Tutorial.', + [':href' => 'http://jmespath.org/tutorial.html'] + ), + + '#ajax' => [ + 'callback' => [$this, 'callJmesPathprocess'], + 'event' => 'change', + 'keypress' => FALSE, + 'disable-refocus' => FALSE, + 'progress' => [ + // Graphic shown to indicate ajax. Options: 'throbber' (default), 'bar'. + 'type' => 'throbber', + ], + ], + '#required' => TRUE, + '#executes_submit_callback' => TRUE, + '#submit' => ['::submitForm'] + ]; + $form['test_jmespath_input'] = [ + '#type' => 'codemirror', + '#codemirror' => $settings, + '#default_value' => $json, + '#rows' => 15, + ]; + $form['test_output'] = [ + '#type' => 'codemirror', + '#prefix' => '
', + '#suffix' => '
', + '#codemirror' => $settings, + '#default_value' => '{}', + '#rows' => 15, + '#attached' => [ + 'library' => [ + 'strawberry_runners/jmespath_codemirror_strawberry_runners', + ], + ], + ]; + } + } + } + } + $form['submit'] = [ + '#type' => 'submit', + '#value' => t('Submit'), + '#attributes' => ['class' => ['js-hide']], + '#submit' => [[$this,'submitForm']] + ]; + return $form; + } + + public function submitForm(array &$form, FormStateInterface $form_state) { + error_log('running Submit'); + $form_state->setRebuild(); + } + + public function callJmesPathprocess(array &$form, FormStateInterface $form_state) { + $response = new AjaxResponse(); + /** @var $itemfield \Drupal\strawberryfield\Plugin\Field\FieldType\StrawberryFieldItem */ + $itemfield = $form_state->get('itemfield'); + try { + $result = $itemfield->searchPath($form_state->getValue('test_jmespath')); + } + catch (\Exception $exception) { + $result = $exception->getMessage(); + } + error_log($form_state->getValue('test_jmespath')); + error_log(var_export($result,true)); + $response->addCommand(new \Drupal\strawberry_runners\Ajax\UpdateCodeMirrorCommand('#jmespathoutput', json_encode($result,JSON_PRETTY_PRINT))); + + return $response; + } + + /** + * Limit access to the Tools according to their restricted state. + * + * @param \Drupal\Core\Session\AccountInterface $account + * The account object. + * @param int $node + * The node id. + */ + public function accessTools(AccountInterface $account, $node) { + $node_storage = $this->entityTypeManager()->getStorage('node'); + $node = $node_storage->load($node); + $type = $node->getType(); + // @TODO for now... + if ($sbf_fields = \Drupal::service('strawberryfield.utility')->bearsStrawberryfield($node)) { + $access = AccessResult::allowedIfHasPermission($account, 'edit any ' . $type . ' content'); + if (!$access->isAllowed() && $account->hasPermission('edit own ' . $type . ' content')) { + $access = $access->orIf(AccessResult::allowedIf($account->id() == $node->getOwnerId())->cachePerUser()->addCacheableDependency($node)); + } + } else { + $access = AccessResult::forbidden(); + } + + $access->addCacheableDependency($node); + return AccessResult::allowedIf($access)->cachePerPermissions(); + + + } + +} diff --git a/src/Form/strawberryRunnerPostprocessorEntityDeleteForm.php b/src/Form/strawberryRunnerPostprocessorEntityDeleteForm.php new file mode 100644 index 0000000..6ad5073 --- /dev/null +++ b/src/Form/strawberryRunnerPostprocessorEntityDeleteForm.php @@ -0,0 +1,56 @@ +t('Are you sure you want to delete %name?', ['%name' => $this->entity->label()]); + } + + /** + * {@inheritdoc} + */ + public function getCancelUrl() { + return new Url('entity.strawberry_runners_postprocessor.collection'); + } + + /** + * {@inheritdoc} + */ + public function getConfirmText() { + return $this->t('Delete'); + } + + /** + * {@inheritdoc} + */ + public function submitForm(array &$form, FormStateInterface $form_state) { + // @TODO we can not delete a Processor that is running + // Make sure we have a drupal state variable with the whole stack + // and block delete if its still running. + $this->entity->delete(); + + $this->messenger()->addMessage( + $this->t('content @type: deleted @label.', + [ + '@type' => $this->entity->bundle(), + '@label' => $this->entity->label(), + ] + ) + ); + + $form_state->setRedirectUrl($this->getCancelUrl()); + } + +} diff --git a/src/Form/strawberryRunnerPostprocessorEntityForm.php b/src/Form/strawberryRunnerPostprocessorEntityForm.php new file mode 100644 index 0000000..676c941 --- /dev/null +++ b/src/Form/strawberryRunnerPostprocessorEntityForm.php @@ -0,0 +1,167 @@ +strawberryRunnersPostProcessorPluginManager = $strawberryRunnersPostProcessorPluginManager; + } + + public static function create(ContainerInterface $container) { + return new static( + $container->get('strawberry_runner.processor_manager') + ); + } + + /** + * {@inheritdoc} + */ + public function form(array $form, FormStateInterface $form_state) { + $form = parent::form($form, $form_state); + + /* @var strawberryRunnerPostprocessorEntity $strawberry_processor */ + $strawberry_processor = $this->entity; + + $form['label'] = [ + '#type' => 'textfield', + '#title' => $this->t('Label'), + '#maxlength' => 255, + '#default_value' => $strawberry_processor->label(), + '#description' => $this->t("Label for this Processor"), + '#required' => TRUE, + ]; + + $form['id'] = [ + '#type' => 'machine_name', + '#default_value' => $strawberry_processor->id(), + '#machine_name' => [ + 'exists' => '\Drupal\strawberry_runners\Entity\strawberryRunnerPostprocessorEntity::load', + ], + '#disabled' => !$strawberry_processor->isNew(), + ]; + + $ajax = [ + 'callback' => [get_class($this), 'ajaxCallback'], + 'wrapper' => 'postprocessorentity-ajax-container', + ]; + /* @var \Drupal\strawberryfield\Plugin\StrawberryfieldKeyNameProviderManager $keyprovider_plugin_definitions */ + $plugin_definitions = $this->strawberryRunnersPostProcessorPluginManager->getDefinitions(); + foreach ($plugin_definitions as $id => $definition) { + $options[$id] = $definition['label']; + } + + $form['pluginid'] = [ + '#type' => 'select', + '#title' => $this->t('Strawberry Runner Post Processor Plugin'), + '#default_value' => $strawberry_processor->getPluginid(), + '#options' => $options, + "#empty_option" =>t('- Select One -'), + '#required'=> true, + '#ajax' => $ajax + ]; + + $form['container'] = [ + '#type' => 'container', + '#attributes' => ['id' => 'postprocessorentity-ajax-container'], + '#weight' => 100, + '#tree' => true + ]; + + $pluginid = $form_state->getValue('pluginid')?:$strawberry_processor->getPluginid(); + if (!empty($pluginid)) { + $this->messenger()->addMessage($form_state->getValue('pluginid')); + $form['container']['pluginconfig'] = [ + '#type' => 'container', + '#parents' => ['pluginconfig'] + ]; + $parents = ['container','pluginconfig']; + $elements = $this->strawberryRunnersPostProcessorPluginManager->createInstance($pluginid,[])->settingsForm($parents, $form_state); + $pluginconfig = $strawberry_processor->getPluginconfig(); + + $form['container']['pluginconfig'] = array_merge($form['container']['pluginconfig'],$elements); + if (!empty($pluginconfig)) { + foreach ($pluginconfig as $key => $value) { + if (isset($form['container']['pluginconfig'][$key])) { + ($form['container']['pluginconfig'][$key]['#default_value'] = $value); + } + } + } + } else { + $form['container']['pluginconfig'] = [ + '#type' => 'container', + ]; + + } + + $form['active'] = [ + '#type' => 'checkbox', + '#title' => $this->t('Is this processor plugin active?'), + '#return_value' => TRUE, + '#default_value' => $strawberry_processor->isActive(), + ]; + + //@TODO allow a preview of the processing via ajax + + return $form; + } + + /** + * {@inheritdoc} + */ + public function save(array $form, FormStateInterface $form_state) { + $strawberry_processor = $this->entity; + + $status = $strawberry_processor->save(); + + switch ($status) { + case SAVED_NEW: + $this->messenger->addStatus($this->t('Created the %label Strawberry Runner Post Processor.', [ + '%label' => $strawberry_processor->label(), + ])); + break; + + default: + $this->messenger->addStatus($this->t('Saved the %label Strawberry Runner Post Processor.', [ + '%label' => $strawberry_processor->label(), + ])); + } + $form_state->setRedirectUrl($strawberry_processor->toUrl('collection')); + } + + /** + * Ajax callback. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + * + * @return array + * An associative array containing entity reference details element. + */ + public static function ajaxCallback(array $form, FormStateInterface $form_state) { + $form_state->setRebuild(); + return $form['container']; + } + + +} diff --git a/src/Plugin/QueueWorker/ProcessWebhookPayloadQueueWorker.php b/src/Plugin/QueueWorker/ProcessWebhookPayloadQueueWorker.php new file mode 100644 index 0000000..0f4cbd1 --- /dev/null +++ b/src/Plugin/QueueWorker/ProcessWebhookPayloadQueueWorker.php @@ -0,0 +1,140 @@ +entityTypeManager = $entity_type_manager; + } + + /** + * Implementation of the container interface to allow dependency injection. + * + * @param \Symfony\Component\DependencyInjection\ContainerInterface $container + * @param array $configuration + * @param string $plugin_id + * @param mixed $plugin_definition + * + * @return static + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + return new static( + empty($configuration) ? [] : $configuration, + $plugin_id, + $plugin_definition, + $container->get('entity_type.manager') + ); + } + + /** + * {@inheritdoc} + */ + public function processItem($data) { + // $data will have S3 API 2.1 Event structure + /* { + "Records":[ + { + "eventVersion":"2.1", + "eventSource":"aws:s3", + "awsRegion":"us-west-2", + "eventTime":The time, in ISO-8601 format, for example, 1970-01-01T00:00:00.000Z, when Amazon S3 finished processing the request, + "eventName":"event-type", + "userIdentity":{ + "principalId":"Amazon-customer-ID-of-the-user-who-caused-the-event" + }, + "requestParameters":{ + "sourceIPAddress":"ip-address-where-request-came-from" + }, + "responseElements":{ + "x-amz-request-id":"Amazon S3 generated request ID", + "x-amz-id-2":"Amazon S3 host that processed the request" + }, + "s3":{ + "s3SchemaVersion":"1.0", + "configurationId":"ID found in the bucket notification configuration", + "bucket":{ + "name":"bucket-name", + "ownerIdentity":{ + "principalId":"Amazon-customer-ID-of-the-bucket-owner" + }, + "arn":"bucket-ARN" + }, + "object":{ + "key":"object-key", + "size":object-size, + "eTag":"object eTag", + "versionId":"object version if bucket is versioning-enabled, otherwise null", + "sequencer": "a string representation of a hexadecimal value used to determine event sequence, + only used with PUTs and DELETEs" + } + }, + "glacierEventData": { + "restoreEventData": { + "lifecycleRestorationExpiryTime": "The time, in ISO-8601 format, for example, 1970-01-01T00:00:00.000Z, of Restore Expiry", + "lifecycleRestoreStorageClass": "Source storage class for restore" + } + } + } + ] +} + */ + + + // Decode the JSON that was captured. + $decode = Json::decode($data); + /* + // Pull out applicable values. + // You may want to do more validation! + $nodeValues = [ + 'type' => 'machine_name_here', + 'status' => 1, + 'title' => $decode['title'], + 'field_custom_field' => $decode['something'], + ]; + + // Create a node. + $storage = $this->entityTypeManager->getStorage('node'); + $node = $storage->create($nodeValues); + $node->save();*/ + + } + +} \ No newline at end of file diff --git a/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php b/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php new file mode 100644 index 0000000..8989abb --- /dev/null +++ b/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php @@ -0,0 +1,226 @@ + 'asstructure', + 'mime_type' => 'application/pdf', + 'path' => '', + 'arguments' => '', + 'output_type' => 'json', + 'output_destination' => 'subkey', + ] + parent::defaultConfiguration(); + } + + + public function calculateDependencies() { + // Since Processors could be chained we need to check if any other + // processor instance is using an instance of this one + // @TODO: Implement calculateDependencies() method. + } + + public function settingsForm(array $parents, FormStateInterface $form_state) { + + $element['source_type'] = [ + '#type' => 'select', + '#title' => $this->t('The type of source data this processor works on'), + '#options' => [ + 'asstructure' => 'File entities referenced in the as:filetype JSON structure', + 'filepath' => 'Full file paths passed by another processor', + ], + '#default_value' => $this->getConfiguration()['source_type'], + '#description' => $this->t('Select from where the source file this processor needs is fetched'), + '#required' => TRUE + ]; + + $element['ado_type'] = [ + '#type' => 'textfield', + '#title' => $this->t('ADO type(s) to limit this processor to.'), + '#default_value' => $this->getConfiguration()['ado_type'], + '#description' => $this->t('A single ADO type or a coma separed list of ado types that qualify to be Processed. Leave empty to apply to all ADOs.'), + ]; + + $element['jsonkey'] = [ + '#type' => 'checkboxes', + '#title' => $this->t('The JSON key that contains the desired source files.'), + '#options' => [ + 'as:image' => 'as:image', + 'as:document' => 'as:document', + 'as:audio' => 'as:audio', + 'as:video' => 'as:video', + 'as:text' => 'as:text', + 'as:application' => 'as:application', + ], + '#default_value' => (!empty($this->getConfiguration()['jsonkey']) && is_array($this->getConfiguration()['jsonkey'])) ? $this->getConfiguration()['jsonkey'] : [], + '#states' => [ + 'visible' => [ + ':input[name="pluginconfig[source_type]"]' => ['value' => 'asstructure'], + ], + ], + '#required' => TRUE, + ]; + + $element['mime_type'] = [ + '#type' => 'textfield', + '#title' => $this->t('Mimetypes(s) to limit this Processor to.'), + '#default_value' => $this->getConfiguration()['mime_type'], + '#description' => $this->t('A single Mimetype type or a coma separed list of mimetypes that qualify to be Processed. Leave empty to apply any file'), + ]; + $element['path'] = [ + '#type' => 'textfield', + '#title' => $this->t('The system path to the binary that will be executed by this processor.'), + '#default_value' => $this->getConfiguration()['path'], + '#description' => t('A full system path to a binary present in the same environment your PHP runs, e.g /usr/local/bin/exif'), + '#required' => TRUE, + ]; + + $element['arguments'] = [ + '#type' => 'textfield', + '#title' => $this->t('Any additional argument your executable binary requires.'), + '#default_value' => !empty($this->getConfiguration()['arguments']) ? $this->getConfiguration()['arguments'] : '%file', + '#description' => t('Any arguments your binary requires to run. Use %file as replacement for the file if the executable requires the filename to be passed under a specific argument.'), + '#required' => TRUE, + ]; + + $element['output_type'] = [ + '#type' => 'select', + '#title' => $this->t('The expected and desired output of this processor.'), + '#options' => [ + 'entity:file' => 'One or more Files', + 'json' => 'Data/Values that can be serialized to JSON', + ], + '#default_value' => $this->getConfiguration()['output_type'], + '#description' => $this->t('If the output is just data and "One or more Files" is selected all data will be dumped into a file and handled as such.'), + ]; + + $element['output_destination'] = [ + '#type' => 'checkboxes', + '#title' => $this->t("Where and how the output will be used."), + '#options' => [ + 'subkey' => 'In the same Source Metadata, as a child structure of each Processed file', + 'ownkey' => 'In the same Source Metadata but inside its own, top level, "as:flavour" subkey based on the given machine name of the current plugin', + 'plugin' => 'As Input for another processor Plugin', + ], + '#default_value' => (!empty($this->getConfiguration()['output_destination']) && is_array($this->getConfiguration()['output_destination']))? $this->getConfiguration()['output_destination']: [], + '#description' => t('As Input for another processor Plugin will only have an effect if another Processor is setup to consume this ouput.'), + '#required' => TRUE, + ]; + + $element['timeout'] = [ + '#type' => 'number', + '#title' => $this->t('Timeout in seconds for this process.'), + '#default_value' => $this->getConfiguration()['timeout'], + '#description' => $this->t('If the process runs out of time it can still be processed again.'), + '#size' => 2, + '#maxlength' => 2, + '#min' => 1, + ]; + $element['weight'] = [ + '#type' => 'number', + '#title' => $this->t('Order or execution in the global chain.'), + '#default_value' => $this->getConfiguration()['weight'], + ]; + + return $element; + } + + public function onDependencyRemoval(array $dependencies) { + // Since Processors could be chained we need to check if any other + // processor instance is using an instance of this one + return parent::onDependencyRemoval( + $dependencies + ); // TODO: Change the autogenerated stub + } + + /** + * Executes the logic of this plugin given a file path and a context. + * + * @param \stdClass $io + * $io->input needs to contain \Drupal\strawberry_runners\Annotation\StrawberryRunnersPostProcessor::$input_property + * $io->output will contain the result of the processor + * @param string $context + */ + public function run(\stdClass $io, $context = StrawberryRunnersPostProcessorPluginInterface::PROCESS) { + // Specific input key as defined in the annotation + // In this case it will contain an absolute Path to a File. + // Needed since this executes locally on the server via SHELL. + $input_property = $this->pluginDefinition['input_property']; + if (isset($io->input->{$input_property})) { + $execstring = $this->buildExecutableCommand($io->input->{$input_property}); + if ($execstring) { + $io->output = $execstring; + dpm($execstring); + } + } else { + throwException(new \InvalidArgumentException); + } + } + + /** + * Builds a clean Command string using a File path. + * + * @param string $filepath + * + * @return null|string + */ + public function buildExecutableCommand(string $filepath) { + $config = $this->getConfiguration(); + $execpath = $config['path']; + $arguments = $config['arguments']; + $command = ''; + if ($this->verifyCommand($execpath) && (strpos($arguments, '%file' ) !== false)) { + $arguments = str_replace('%s','', $arguments); + $arguments = str_replace_first('%file','%s', $arguments); + $arguments = sprintf($arguments, $filepath); + $command = escapeshellcmd($execpath.' '.$arguments); + } + // Only return $command if it contains the original filepath somewhere + if (strpos($command, $filepath) !== false) { return $command;} + return ''; + + } + + /** + * Checks if a given command exists and is executable. + * + * @param $command + * + * @return bool + */ + private function verifyCommand($execpath) :bool { + $iswindows = strpos(PHP_OS, 'WIN') === 0; + $execpath = trim(escapeshellcmd($execpath)); + $test = $iswindows ? 'where' : 'command -v'; + return is_executable(shell_exec("$test $execpath")); + } + +} \ No newline at end of file diff --git a/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php b/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php new file mode 100644 index 0000000..fd3ba06 --- /dev/null +++ b/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php @@ -0,0 +1,136 @@ +entityTypeBundleInfo = $entityTypeBundleInfo; + $this->entityTypeManager = $entityTypeManager; + $this->setConfiguration($configuration); + $this->httpClient = $httpClient; + } + + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + + return new static( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('entity_type.manager'), + $container->get('entity_type.bundle.info'), + $container->get('http_client') + ); + } + + /** + * {@inheritdoc} + */ + public function defaultConfiguration() { + return [ + 'jsonkey' => ['as:image'], + 'ado_type' => 'Book', + // Max time to run in seconds per item. + 'timeout' => 10, + // Order in which this processor is executed in the chain + 'weight' => 0, + // The id of the config entity from where these values came from. + 'configEntity' => '', + ]; + } + + /** + * @param array $parents + * @param FormStateInterface $form_state; + * + * @return array + */ + public function settingsForm(array $parents, FormStateInterface $form_state) { + return []; + } + /** + * {@inheritdoc} + */ + public function label() { + $definition = $this->getPluginDefinition(); + // The label can be an object. + // @see \Drupal\Core\StringTranslation\TranslatableMarkup + return $definition['label']; + } + + /** + * {@inheritdoc} + */ + public function setConfiguration(array $configuration) { + $this->configuration = $configuration + $this->defaultConfiguration(); + } + + /** + * {@inheritdoc} + */ + public function getConfiguration() { + return $this->configuration; + } + + /** + * {@inheritdoc} + */ + public function onDependencyRemoval(array $dependencies) { + return FALSE; + } + + /** + * {@inheritdoc} + */ + public function run(\stdClass $io, $context = strawberryRunnersPostProcessorPluginInterface::PROCESS) { + return FALSE; + } + + +} \ No newline at end of file diff --git a/src/Plugin/StrawberryRunnersPostProcessorPluginInterface.php b/src/Plugin/StrawberryRunnersPostProcessorPluginInterface.php new file mode 100644 index 0000000..5e1090a --- /dev/null +++ b/src/Plugin/StrawberryRunnersPostProcessorPluginInterface.php @@ -0,0 +1,72 @@ +input needs to contain \Drupal\strawberry_runners\Annotation\StrawberryRunnersPostProcessor::$input_property + * $io->output will contain the result of the processor + * @param string $context + * Can be either of + * StrawberryRunnersPostProcessorPluginInterface::PROCESS + * StrawberryRunnersPostProcessorPluginInterface::TEST + * StrawberryRunnersPostProcessorPluginInterface::BENCHMARK + * Each plugin needs to know how to deal with this. + * + */ + public function run(\stdClass $io, $context = StrawberryRunnersPostProcessorPluginInterface::PROCESS); + +} \ No newline at end of file diff --git a/src/Plugin/StrawberryRunnersPostProcessorPluginManager.php b/src/Plugin/StrawberryRunnersPostProcessorPluginManager.php new file mode 100644 index 0000000..606247e --- /dev/null +++ b/src/Plugin/StrawberryRunnersPostProcessorPluginManager.php @@ -0,0 +1,43 @@ +alterInfo('strawberry_runners_strawberryrunnerspostprocessor_info'); + $this->setCacheBackend($cache_backend,'strawberry_runners_strawberryrunnerspostprocessor_plugins'); + } + + +} \ No newline at end of file diff --git a/src/strawberryRunnerPostProcessorEntityHtmlRouteProvider.php b/src/strawberryRunnerPostProcessorEntityHtmlRouteProvider.php new file mode 100644 index 0000000..6e02d77 --- /dev/null +++ b/src/strawberryRunnerPostProcessorEntityHtmlRouteProvider.php @@ -0,0 +1,25 @@ += 8.8)' + - 'drupal:user' + - 'strawberryfield' diff --git a/strawberry_runners.install b/strawberry_runners.install new file mode 100644 index 0000000..7731bb2 --- /dev/null +++ b/strawberry_runners.install @@ -0,0 +1,32 @@ +installEntityType(new ConfigEntityType([ + 'id' => 'strawberry_runners_postprocessor', + 'label' => new TranslatableMarkup('Strawberry Runners Post Processor Configuration'), + 'config_prefix' => 'strawberry_runners_postprocessor', + 'admin_permission' => 'administer site configuration', + 'entity_keys' => [ + 'id' => 'id', + 'label' => 'label', + 'uuid' => 'uuid', + 'active' => 'active', + 'weight' => 'active', + ], + 'config_export' => [ + 'id', + 'label', + 'uuid', + 'weight', + 'pluginconfig', + 'pluginid', + 'active' + ], + ])); +} \ No newline at end of file diff --git a/strawberry_runners.libraries.yml b/strawberry_runners.libraries.yml new file mode 100644 index 0000000..b3e0d4b --- /dev/null +++ b/strawberry_runners.libraries.yml @@ -0,0 +1,7 @@ +jmespath_codemirror_strawberry_runners: + version: 1.0 + js: + js/jmespath-codemirror_strawberry_runners.js: {minified: false} + dependencies: + - core/jquery + - core/drupal \ No newline at end of file diff --git a/strawberry_runners.links.action.yml b/strawberry_runners.links.action.yml new file mode 100644 index 0000000..920e25f --- /dev/null +++ b/strawberry_runners.links.action.yml @@ -0,0 +1,6 @@ +#Adds the Archipelago Runners Post Processor Plugin add +entity.strawberry_runners_postprocessor.add_form: + route_name: 'entity.strawberry_runners_postprocessor.add_form' + title: 'Add a new Strawberry Runner Post Processor' + appears_on: + - entity.strawberry_runners_postprocessor.collection \ No newline at end of file diff --git a/strawberry_runners.links.menu.yml b/strawberry_runners.links.menu.yml new file mode 100644 index 0000000..19b3ba2 --- /dev/null +++ b/strawberry_runners.links.menu.yml @@ -0,0 +1,7 @@ +#Adds the Archipelago Runners Post Processor Config +entity.strawberry_runners_postprocessor.collection: + title: 'Configure Strawberry Runners Post Processors' + route_name: entity.strawberry_runners_postprocessor.collection + description: 'Configure what Post Processor will run for ADOs' + parent: strawberryfield.group.admin + weight: 200 \ No newline at end of file diff --git a/strawberry_runners.links.task.yml b/strawberry_runners.links.task.yml new file mode 100644 index 0000000..a0c581f --- /dev/null +++ b/strawberry_runners.links.task.yml @@ -0,0 +1,4 @@ +strawberry_runners.ado_tools: + route_name: strawberry_runners.ado_tools + base_route: entity.node.canonical + title: 'ADO Tools' \ No newline at end of file diff --git a/strawberry_runners.module b/strawberry_runners.module new file mode 100644 index 0000000..1673d31 --- /dev/null +++ b/strawberry_runners.module @@ -0,0 +1,11 @@ + Date: Tue, 17 Mar 2020 15:50:50 -0400 Subject: [PATCH 2/6] Address @giancarlobi review. `weight` --- .gitignore | 8 ++++++++ strawberry_runners.install | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a8259d --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ + +.idea/workspace.xml +.idea/vcs.xml +.idea/strawberry_runners.iml +.idea/php.xml +.idea/modules.xml +.idea/misc.xml +.idea/codeStyles/codeStyleConfig.xml diff --git a/strawberry_runners.install b/strawberry_runners.install index 7731bb2..9b61b2e 100644 --- a/strawberry_runners.install +++ b/strawberry_runners.install @@ -17,7 +17,7 @@ function strawberry_runners_update_8801() { 'label' => 'label', 'uuid' => 'uuid', 'active' => 'active', - 'weight' => 'active', + 'weight' => 'weight', ], 'config_export' => [ 'id', From 1cf8cee33fe9625424ab7fb53bf330e2c249ad85 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Wed, 8 Apr 2020 15:36:25 -0400 Subject: [PATCH 3/6] Small fixes. So much more to do yet @giancarlobi not sure why the hook update N is/was not running, i also changed the number because it really ignored it . Maybe its the lack of proper versioning of this module yet? So do have it running i did this drush eval "module_load_install('strawberry_runners'); strawberry_runners_update_8100();" --- ...sEventPreSavePostProcessingSubscriber.php} | 50 +++++++++++-------- strawberry_runners.install | 8 +-- strawberry_runners.services.yml | 7 ++- 3 files changed, 40 insertions(+), 25 deletions(-) rename src/EventSubscriber/{StrawberryRunnersEventJsonProcessingSubscriber.php => StrawberryRunnersEventPreSavePostProcessingSubscriber.php} (86%) diff --git a/src/EventSubscriber/StrawberryRunnersEventJsonProcessingSubscriber.php b/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php similarity index 86% rename from src/EventSubscriber/StrawberryRunnersEventJsonProcessingSubscriber.php rename to src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php index 12bc28b..32af4b3 100644 --- a/src/EventSubscriber/StrawberryRunnersEventJsonProcessingSubscriber.php +++ b/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php @@ -3,11 +3,10 @@ namespace Drupal\strawberry_runners\EventSubscriber; use Drupal\Core\Entity\EntityTypeManagerInterface; -use Drupal\strawberryfield\Event\StrawberryfieldServiceEvent; -use Drupal\strawberryfield\EventSubscriber\StrawberryfieldEventJsonProcessingSubscriber; +use Drupal\strawberryfield\Event\StrawberryfieldCrudEvent; +use Drupal\strawberryfield\EventSubscriber\StrawberryfieldEventPresaveSubscriber; use Drupal\Core\StringTranslation\StringTranslationTrait; use Drupal\Core\StringTranslation\TranslationInterface; -use Drupal\Core\Entity\EntityStorageInterface; use Drupal\Core\Messenger\MessengerInterface; use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Logger\LoggerChannelFactoryInterface; @@ -21,15 +20,18 @@ /** * Event subscriber for SBF bearing entity json process event. */ -class StrawberryRunnersEventJsonProcessingSubscriber extends StrawberryfieldEventJsonProcessingSubscriber { +class StrawberryRunnersEventPreSavePostProcessingSubscriber extends StrawberryfieldEventPresaveSubscriber { use StringTranslationTrait; /** + * + * Run as late as possible. + * * @var int */ - protected static $priority = -700; + protected static $priority = -2000; /** * The messenger. @@ -98,12 +100,10 @@ class StrawberryRunnersEventJsonProcessingSubscriber extends StrawberryfieldEven protected $strawberryRunnerProcessorPluginManager; /** - * StrawberryRunnersEventJsonProcessingSubscriber constructor. + * StrawberryRunnersEventPreSavePostProcessingSubscriber constructor. * - * @param \Drupal\Core\Entity\EntityStorageInterface $storage * @param \Drupal\Core\StringTranslation\TranslationInterface $string_translation * @param \Drupal\Core\Messenger\MessengerInterface $messenger - * @param \Symfony\Component\Serializer\SerializerInterface $serializer * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory * @param \Drupal\Core\Logger\LoggerChannelFactoryInterface $logger_factory * @param \Drupal\Core\StreamWrapper\StreamWrapperManagerInterface $stream_wrapper_manager @@ -111,34 +111,35 @@ class StrawberryRunnersEventJsonProcessingSubscriber extends StrawberryfieldEven * @param \Drupal\strawberry_runners\Plugin\StrawberryRunnersPostProcessorPluginManager $strawberry_runner_processor_plugin_manager */ public function __construct( - EntityTypeManagerInterface $entity_type_manager, - EntityStorageInterface $storage, TranslationInterface $string_translation, MessengerInterface $messenger, - ConfigFactoryInterface $config_factory, LoggerChannelFactoryInterface $logger_factory, + ConfigFactoryInterface $config_factory, StreamWrapperManagerInterface $stream_wrapper_manager, FileSystemInterface $file_system, + EntityTypeManagerInterface $entity_type_manager, StrawberryRunnersPostProcessorPluginManager $strawberry_runner_processor_plugin_manager ) { $this->stringTranslation = $string_translation; $this->messenger = $messenger; $this->loggerFactory = $logger_factory; $this->configFactory = $config_factory; - $this->storage = $storage; - $this->entityTypeManager = $entity_type_manager; $this->streamWrapperManager = $stream_wrapper_manager; $this->fileSystem = $file_system; + $this->entityTypeManager = $entity_type_manager; $this->strawberryRunnerProcessorPluginManager = $strawberry_runner_processor_plugin_manager; } /** - * Method called when Event occurs. + * Method called when Event occurs. * - * @param \Drupal\strawberryfield\Event\StrawberryfieldServiceEvent $event - * The event. + * @param \Drupal\strawberryfield\Event\StrawberryfieldCrudEvent $event + * + * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException + * @throws \Drupal\Component\Plugin\Exception\PluginException + * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - public function onJsonInvokeProcess(StrawberryfieldServiceEvent $event) { + public function onEntityPresave(StrawberryfieldCrudEvent $event) { /* @var $plugin_config_entities \Drupal\strawberry_runners\Entity\strawberryRunnerPostprocessorEntity[] */ $plugin_config_entities = $this->entityTypeManager->getListBuilder('strawberry_runners_postprocessor')->load(); @@ -189,8 +190,9 @@ public function onJsonInvokeProcess(StrawberryfieldServiceEvent $event) { ); /** @var $file FileInterface; */ if ($file) { - $this->add_file_usage($file, $entity->id(), $entity_type_id); - $updated++; + //$this->add_file_usage($file, $entity->id(), $entity_type_id); + //$updated++; + $this->processFile($file); } else { $this->messenger()->addError( @@ -206,6 +208,10 @@ public function onJsonInvokeProcess(StrawberryfieldServiceEvent $event) { } } } + $current_class = get_called_class(); + $event->setProcessedBy($current_class, TRUE); + $this->messenger->addStatus(t('Post processor was invoked')); + } /** @@ -230,7 +236,7 @@ private function processFile(FileInterface $file) { // Local stream. $cache_key = md5($uri); if (empty($this->instanceCopiesOfFiles[$cache_key])) { - if (!($this->instanceCopiesOfFiles[$cache_key] = $this->fileSystem->copy($uri, 'temporary://sbr_' . $cache_key . '_' . basename($uri), FileSystemInterface::FILE_EXISTS_REPLACE))) { + if (!($this->instanceCopiesOfFiles[$cache_key] = $this->fileSystem->copy($uri, 'temporary://sbr_' . $cache_key . '_' . basename($uri), FileSystemInterface::EXISTS_REPLACE))) { $this->loggerFactory->get('strawberry_runners') ->notice('Unable to create local temporary copy of remote file for Strawberry Runners Post processing File %file.', [ @@ -267,7 +273,9 @@ private function sanitizeValue($string) { public function __destruct() { // Get rid of temporary files created for this instance. foreach ($this->instanceCopiesOfFiles as $uri) { - \Drupal::service('file_system')->unlink($uri); + error_log('should destroy '.$uri); + error_log('better to keep usage count?'); + // \Drupal::service('file_system')->unlink($uri); } } diff --git a/strawberry_runners.install b/strawberry_runners.install index 9b61b2e..f708266 100644 --- a/strawberry_runners.install +++ b/strawberry_runners.install @@ -1,12 +1,14 @@ installEntityType(new ConfigEntityType([ 'id' => 'strawberry_runners_postprocessor', 'label' => new TranslatableMarkup('Strawberry Runners Post Processor Configuration'), @@ -29,4 +31,4 @@ function strawberry_runners_update_8801() { 'active' ], ])); -} \ No newline at end of file +} diff --git a/strawberry_runners.services.yml b/strawberry_runners.services.yml index ae8bfe2..bb3f518 100644 --- a/strawberry_runners.services.yml +++ b/strawberry_runners.services.yml @@ -1,4 +1,9 @@ services: strawberry_runner.processor_manager: class: Drupal\strawberry_runners\Plugin\StrawberryRunnersPostProcessorPluginManager - parent: default_plugin_manager \ No newline at end of file + parent: default_plugin_manager + strawberry_runner.postprocessing_subscriber: + class: Drupal\strawberry_runners\EventSubscriber\StrawberryRunnersEventPreSavePostProcessingSubscriber + tags: + - {name: event_subscriber} + arguments: ['@string_translation', '@messenger', '@logger.factory', '@config.factory', '@stream_wrapper_manager', '@file_system', '@entity_type.manager', '@strawberry_runner.processor_manager'] \ No newline at end of file From f262de0a4ea4d0a75c89dd7c756bbf6d39c26a83 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Thu, 9 Apr 2020 15:17:15 -0400 Subject: [PATCH 4/6] Initial steps of some a React Service This are placeholder files for a Drupal Service that will replace src/Scripts/mainLoop.php Idea here is that Setting up the queue, checking if the loop is already running or not, etc, is all handled by this service. Which in turn will have some methods invoked by \Drupal\strawberry_runners\EventSubscriber\StrawberryRunnersEventPreSavePostProcessingSubscriber once code is more advanced All this are just basic dependencies, service definition and some high level ideas... but this is going to be AWESOME! =) --- composer.json | 9 +- src/StrawberryRunnersLoopService.php | 164 +++++++++++++++++++++++++++ strawberry_runners.services.yml | 6 +- 3 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 src/StrawberryRunnersLoopService.php diff --git a/composer.json b/composer.json index e80b70f..6e2fdaa 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "strawberryfield/strawberry_runners", - "description": "Strawberryfield post processing module for Drupal 8", + "description": "Strawberryfield post processing module for Drupal 8 using PHP React", "type": "drupal-module", "license": "GPL-2.0+", "homepage": "https://github.com/esmero/strawberry_runners", @@ -20,7 +20,10 @@ ], "require": { "ml/json-ld": "^1.0", - "mtdowling/jmespath.php":"^2.4", - "strawberryfield/strawberryfield":"dev-8.x-1.0-beta2" + "mtdowling/jmespath.php": "^2.4", + "strawberryfield/strawberryfield": "dev-8.x-1.0-beta2", + "react/event-loop": "^1.1", + "react/child-process": "^0.6", + "react/stream": "^1.1" } } diff --git a/src/StrawberryRunnersLoopService.php b/src/StrawberryRunnersLoopService.php new file mode 100644 index 0000000..85eceee --- /dev/null +++ b/src/StrawberryRunnersLoopService.php @@ -0,0 +1,164 @@ +fileSystem = $file_system; + $this->entityTypeManager = $entity_type_manager; + $this->configFactory = $config_factory; + $this->moduleHandler = $module_handler; + $this->lock = $lock; + $this->queueFactory = $queue_factory; + $this->state = $state; + $this->accountSwitcher = $account_switcher; + $this->logger = $logger; + $this->queueManager = $queue_manager; + $this->time = $time ?: \Drupal::service('datetime.time'); + } + + + +public function mainLoop() { + +} + + +} diff --git a/strawberry_runners.services.yml b/strawberry_runners.services.yml index bb3f518..9d182de 100644 --- a/strawberry_runners.services.yml +++ b/strawberry_runners.services.yml @@ -6,4 +6,8 @@ services: class: Drupal\strawberry_runners\EventSubscriber\StrawberryRunnersEventPreSavePostProcessingSubscriber tags: - {name: event_subscriber} - arguments: ['@string_translation', '@messenger', '@logger.factory', '@config.factory', '@stream_wrapper_manager', '@file_system', '@entity_type.manager', '@strawberry_runner.processor_manager'] \ No newline at end of file + arguments: ['@string_translation', '@messenger', '@logger.factory', '@config.factory', '@stream_wrapper_manager', '@file_system', '@entity_type.manager', '@strawberry_runner.processor_manager'] + strawberry_runner.loop: + class: Drupal\strawberry_runners\StrawberryRunnersLoopService + arguments: ['@file_system', '@entity_type.manager', '@config.factory', '@module_handler', '@lock', '@queue', '@state', '@account_switcher', '@logger.factory', '@plugin.manager.queue_worker', '@datetime.time'] +] From a4aef75d541b4dda3c6486119d1d80a4176dcd99 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Tue, 29 Sep 2020 08:42:11 -0400 Subject: [PATCH 5/6] Refactor and add mainLoop() as a Service --- ...rsEventPreSavePostProcessingSubscriber.php | 66 +++++++++++++++++-- .../SystemBinaryPostProcessor.php | 2 +- src/StrawberryRunnersLoopService.php | 22 ++++++- strawberry_runners.services.yml | 3 +- 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php b/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php index 32af4b3..51c56ab 100644 --- a/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php +++ b/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php @@ -74,7 +74,7 @@ class StrawberryRunnersEventPreSavePostProcessingSubscriber extends Strawberryfi * * @var array */ - protected $instanceCopiesOfFiles; + protected $instanceCopiesOfFiles = []; /** * File system service. @@ -91,7 +91,6 @@ class StrawberryRunnersEventPreSavePostProcessingSubscriber extends Strawberryfi protected $loggerFactory; - /** * The StrawberryRunner Processor Plugin Manager. * @@ -157,14 +156,16 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { $plugin_definition = $plugin_instance->getPluginDefinition(); // We don't use the key here to preserve the original weight given order // Classify by input type - $active_plugins[$plugin_definition['input_type']][] = $plugin_instance; + + //dpm($plugin_instance); + $active_plugins[$plugin_definition['input_type']][$entity_id] = $plugin_instance->getConfiguration(); } } // We will fetch all files and then see if each file can be processed by one // or more plugin. // Slower option would be to traverse every file per processor. - + //dpm($active_plugins); $updated = 0; $entity = $event->getEntity(); $sbf_fields = $event->getFields(); @@ -192,7 +193,7 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { if ($file) { //$this->add_file_usage($file, $entity->id(), $entity_type_id); //$updated++; - $this->processFile($file); + $this->ensureFileAvailabilty($file); } else { $this->messenger()->addError( @@ -210,7 +211,7 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { } $current_class = get_called_class(); $event->setProcessedBy($current_class, TRUE); - $this->messenger->addStatus(t('Post processor was invoked')); + //$this->messenger->addStatus(t('Post processor was invoked')); } @@ -223,7 +224,7 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { * @return array * Output of processing chain for a particular file. */ - private function processFile(FileInterface $file) { + private function ensureFileAvailabilty(FileInterface $file) { $uri = $file->getFileUri(); $processOutput = []; @@ -279,6 +280,57 @@ public function __destruct() { } } + /** + * Adds File usage to DB for temp files used by SB Runners. + * + * This differs from how we count managed files in other places like SBF. + * Every Post Processor that needs the file will add a count + * Once done, will remove one. File will become unused when everyone releases it. + * + * + * @param \Drupal\file\FileInterface $file + * @param int $nodeid + */ + protected function add_file_usage(FileInterface $file, int $nodeid, string $entity_type_id = 'node') { + if (!$file || !$this->moduleHandler->moduleExists('file')) { + return; + } + /** @var \Drupal\file\FileUsage\FileUsageInterface $file_usage */ + + if ($file) { + $this->fileUsage->add($file, 'strawberry_runners', $entity_type_id, $nodeid); + } + } + + /** + * Deletes File usage from DB for temp files used by SB Runners. + * + * @param \Drupal\file\FileInterface $file + * @param int $nodeid + * @param int $count + * If count is 0 it will remove all references. + */ + protected function remove_file_usage( + FileInterface $file, + int $nodeid, + string $entity_type_id = 'node', + $count = 1 + ) { + if (!$file || !$this->moduleHandler->moduleExists('file')) { + return; + } + /** @var \Drupal\file\FileUsage\FileUsageInterface $file_usage */ + + if ($file) { + $this->fileUsage->delete( + $file, + 'strawberry_runners', + $entity_type_id, + $nodeid, + $count + ); + } + } diff --git a/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php b/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php index 8989abb..7749426 100644 --- a/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php +++ b/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php @@ -178,7 +178,7 @@ public function run(\stdClass $io, $context = StrawberryRunnersPostProcessorPlug $execstring = $this->buildExecutableCommand($io->input->{$input_property}); if ($execstring) { $io->output = $execstring; - dpm($execstring); + //dpm($execstring); } } else { throwException(new \InvalidArgumentException); diff --git a/src/StrawberryRunnersLoopService.php b/src/StrawberryRunnersLoopService.php index 85eceee..4a7ce63 100644 --- a/src/StrawberryRunnersLoopService.php +++ b/src/StrawberryRunnersLoopService.php @@ -156,9 +156,27 @@ public function __construct( -public function mainLoop() { + public function mainLoop() { + + } + + /** + * Push an Item on 'strawberry_runners' queue + */ + public function pushItemOnQueue($node_id, $jsondata, $flavour) { + $element[0] = $node_id; + $element[1] = $jsondata; + $element[2] = $flavour; + //add element to queue + echo 'Push 1 item on queue' . PHP_EOL; + + $queue = $this->queueFactory->get('strawberry_runners', TRUE); + $queue->createItem(serialize($element)); + + $totalItems = $queue->numberOfItems(); + echo 'TotalItems on queue ' . $totalItems . PHP_EOL; + } -} } diff --git a/strawberry_runners.services.yml b/strawberry_runners.services.yml index 9d182de..de8d497 100644 --- a/strawberry_runners.services.yml +++ b/strawberry_runners.services.yml @@ -9,5 +9,4 @@ services: arguments: ['@string_translation', '@messenger', '@logger.factory', '@config.factory', '@stream_wrapper_manager', '@file_system', '@entity_type.manager', '@strawberry_runner.processor_manager'] strawberry_runner.loop: class: Drupal\strawberry_runners\StrawberryRunnersLoopService - arguments: ['@file_system', '@entity_type.manager', '@config.factory', '@module_handler', '@lock', '@queue', '@state', '@account_switcher', '@logger.factory', '@plugin.manager.queue_worker', '@datetime.time'] -] + arguments: ['@file_system', '@entity_type.manager', '@config.factory', '@module_handler', '@lock', '@queue', '@state', '@account_switcher', '@logger.factory', '@plugin.manager.queue_worker', '@datetime.time'] \ No newline at end of file From 8f238b77bb4293ffc147d01aa08beebc84c1d45a Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Thu, 1 Oct 2020 22:56:13 -0400 Subject: [PATCH 6/6] Finally! SB Runner (binary) generates a new Solr Document of Flavour Data Source? So what is this? I would say quite a success but not ready for production yet. Took me forever! This adds now: - An event subscriber (presave) that knows how to handle File based processing using the rules that Sysmte Binary Config Entity sets - System Binary (generic) SB Runner Processor (if you want to test ask me how!, will document as soon as i do the other 27 things i need for tomorrow!) - A Queue worker that actually does the work. Means reads the files, copies them to local place, executes ::run method, puts the output into a key/value and then indexes the new StrawberryfieldFlavorDatasource Item! So what is this wonder? Fun thing is with this we can search in parallell to a normal entity, e.g PDF hightlighs, or URLs inside a WARC, or basically anything that is 1 node to many files to many many sources. Depends on: changes on SBF to make our DataSource Smarter TODO?: A lot 1.- avoid reindexing if the file/output is already in Solr. How do we check? we need to add the Checksum of the file + the signature of the processor. If all matches then all is == and reindexng is not needed 2.- Once all processing is done = package all inside a Data package (zip with info.json and attach to the original NODE as an as:flavor. that way reindexing requires no processing at all (still a copy/and download). 3.- On file removal also remove the new data sources. Means we need to act on file delete to remove our new Solr Documents 4.- Allow settings / ways (need to think of) so user configured Processors can generate multiple outputs. Right now to make that happen we have to provide a fixed/not configurable Processor Plugin --- .../StrawberryRunnersPostProcessor.php | 13 + ...rsEventPreSavePostProcessingSubscriber.php | 147 ++++----- .../IndexPostProcessorQueueWorker.php | 308 ++++++++++++++++++ .../SystemBinaryPostProcessor.php | 31 +- ...rawberryRunnersPostProcessorPluginBase.php | 5 +- .../SbrWarctotextExtractorRemote.php | 133 ++++++++ src/StrawberryRunnersLoopService.php | 1 + strawberry_runners.module | 57 ++++ strawberry_runners.services.yml | 7 +- 9 files changed, 617 insertions(+), 85 deletions(-) create mode 100644 src/Plugin/QueueWorker/IndexPostProcessorQueueWorker.php create mode 100644 src/Plugin/search_api_attachments/SbrWarctotextExtractorRemote.php diff --git a/src/Annotation/StrawberryRunnersPostProcessor.php b/src/Annotation/StrawberryRunnersPostProcessor.php index 843d5aa..410c7df 100644 --- a/src/Annotation/StrawberryRunnersPostProcessor.php +++ b/src/Annotation/StrawberryRunnersPostProcessor.php @@ -20,6 +20,10 @@ */ class StrawberryRunnersPostProcessor extends Plugin { + const PRESAVE = 'preSave'; + const INDEX = 'search_api'; + + /** * The plugin id. * @@ -51,4 +55,13 @@ class StrawberryRunnersPostProcessor extends Plugin { */ public $input_property; + + /** + * Processing stage: can be Entity PreSave or Index tme search_api + * + * @var string $when; + * + */ + public $when = StrawberryRunnersPostProcessor::PRESAVE; + } \ No newline at end of file diff --git a/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php b/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php index 51c56ab..f0b341b 100644 --- a/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php +++ b/src/EventSubscriber/StrawberryRunnersEventPreSavePostProcessingSubscriber.php @@ -157,7 +157,6 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { // We don't use the key here to preserve the original weight given order // Classify by input type - //dpm($plugin_instance); $active_plugins[$plugin_definition['input_type']][$entity_id] = $plugin_instance->getConfiguration(); } } @@ -165,11 +164,52 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { // We will fetch all files and then see if each file can be processed by one // or more plugin. // Slower option would be to traverse every file per processor. - //dpm($active_plugins); - $updated = 0; + + $entity = $event->getEntity(); $sbf_fields = $event->getFields(); - $processedcount = 0; + + + // First pass: for files, all the as:structures we want for, keyed by content type + /* check your config + "source_type" => "asstructure" + "ado_type" => "Document" + "jsonkey" => array:6 [▼ + "as:document" => "as:document" + "as:image" => 0 + "as:audio" => 0 + "as:video" => 0 + "as:text" => 0 + "as:application" => 0 + ] + "mime_type" => "application/pdf" + "path" => "/usr/bin/pdftotext" + "arguments" => "%file" + "output_type" => "json" + "output_destination" => array:3 [▼ + "plugin" => "plugin" + "subkey" => 0 + "ownkey" => 0 + ] + "timeout" => "10" + "weight" => "0" + "configEntity" => "test" + ]*/ + + if (isset($active_plugins['entity:file'])) { + foreach($active_plugins['entity:file'] as $activePluginId => $config) { + if ($config['source_type'] == 'asstructure') { + $askeys = array_filter($config['jsonkey']); + foreach($askeys as $key => $value) { + $askeymap[$key][$activePluginId] = $config; + } + } + } + } + + + + foreach ($sbf_fields as $field_name) { /* @var $field \Drupal\Core\Field\FieldItemInterface */ $field = $entity->get($field_name); @@ -182,26 +222,30 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { /** @var $itemfield \Drupal\strawberryfield\Plugin\Field\FieldType\StrawberryFieldItem */ $flatvalues = (array) $itemfield->provideFlatten(); // Run first on entity:files - - if (isset($flatvalues['dr:fid'])) { - foreach ($flatvalues['dr:fid'] as $fid) { - if (is_numeric($fid)) { - $file = $this->entityTypeManager->getStorage('file')->load( - $fid - ); - /** @var $file FileInterface; */ - if ($file) { - //$this->add_file_usage($file, $entity->id(), $entity_type_id); - //$updated++; - $this->ensureFileAvailabilty($file); - } - else { - $this->messenger()->addError( - t( - 'Your content references a file with Internal ID @file_id that does not exist or was removed.', - ['@file_id' => $fid] - ) - ); + $sbf_type = NULL; + if (isset($flatvalues['type'])) { + $sbf_type = $flatvalues['type']; + } + foreach ($askeymap as $jsonkey => $activePlugins) { + if (isset($flatvalues[$jsonkey])) { + foreach ($flatvalues[$jsonkey] as $uniqueid => $asstructure) { + if (isset($asstructure['dr:fid']) && is_numeric($asstructure['dr:fid'])) { + + foreach($activePlugins as $activePluginId => $config) { + $valid_mimes = []; + if (empty($config['ado_type']) || in_array($config['ado_type'] , $sbf_type)) { + $valid_mimes = explode(',', $config['mime_type']); + if (empty($valid_mimes) || (isset($asstructure["dr:mimetype"]) && in_array($asstructure["dr:mimetype"], $valid_mimes))) { + $data = new \stdClass(); + $data->fid = $asstructure['dr:fid']; + $data->nid = $entity->id(); + $data->plugin_config_entity_id = $activePluginId; + + \Drupal::queue('strawberryrunners_process_index') + ->createItem($data); + } + } + } } } } @@ -211,46 +255,10 @@ public function onEntityPresave(StrawberryfieldCrudEvent $event) { } $current_class = get_called_class(); $event->setProcessedBy($current_class, TRUE); - //$this->messenger->addStatus(t('Post processor was invoked')); + $this->messenger->addStatus(t('Post processor was invoked')); } - /** - * Move file to local and process. - * - * @param \Drupal\file\FileInterface $file - * The File URI to look at. - * - * @return array - * Output of processing chain for a particular file. - */ - private function ensureFileAvailabilty(FileInterface $file) { - $uri = $file->getFileUri(); - $processOutput = []; - - /** @var \Drupal\Core\File\FileSystem $file_system */ - $scheme = $this->fileSystem->uriScheme($uri); - - // If the file isn't stored locally make a temporary copy. - if (!isset($this->streamWrapperManager - ->getWrappers(StreamWrapperInterface::LOCAL)[$scheme])) { - // Local stream. - $cache_key = md5($uri); - if (empty($this->instanceCopiesOfFiles[$cache_key])) { - if (!($this->instanceCopiesOfFiles[$cache_key] = $this->fileSystem->copy($uri, 'temporary://sbr_' . $cache_key . '_' . basename($uri), FileSystemInterface::EXISTS_REPLACE))) { - $this->loggerFactory->get('strawberry_runners') - ->notice('Unable to create local temporary copy of remote file for Strawberry Runners Post processing File %file.', - [ - '%file' => $uri, - ]); - return []; - } - } - $uri = $this->instanceCopiesOfFiles[$cache_key]; - } - - return $processOutput; - } /** * Make sure no HTML or Javascript will be passed around. @@ -268,18 +276,6 @@ private function sanitizeValue($string) { return $string; } - /** - * Cleanup of artifacts from processing files. - */ - public function __destruct() { - // Get rid of temporary files created for this instance. - foreach ($this->instanceCopiesOfFiles as $uri) { - error_log('should destroy '.$uri); - error_log('better to keep usage count?'); - // \Drupal::service('file_system')->unlink($uri); - } - } - /** * Adds File usage to DB for temp files used by SB Runners. * @@ -332,9 +328,4 @@ protected function remove_file_usage( } } - - - - - } diff --git a/src/Plugin/QueueWorker/IndexPostProcessorQueueWorker.php b/src/Plugin/QueueWorker/IndexPostProcessorQueueWorker.php new file mode 100644 index 0000000..4e8e13a --- /dev/null +++ b/src/Plugin/QueueWorker/IndexPostProcessorQueueWorker.php @@ -0,0 +1,308 @@ +entityTypeManager = $entity_type_manager; + $this->strawberryRunnerProcessorPluginManager = $strawberry_runner_processor_plugin_manager; + $this->fileSystem = $file_system; + $this->streamWrapperManager = $stream_wrapper_manager; + $this->keyValue = $key_value; + $this->logger = $logger; + } + + /** + * Implementation of the container interface to allow dependency injection. + * + * @param \Symfony\Component\DependencyInjection\ContainerInterface $container + * @param array $configuration + * @param string $plugin_id + * @param mixed $plugin_definition + * + * @return static + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + return new static( + empty($configuration) ? [] : $configuration, + $plugin_id, + $plugin_definition, + $container->get('entity_type.manager'), + $container->get('strawberry_runner.processor_manager'), + $container->get('file_system'), + $container->get('stream_wrapper_manager'), + $container->get('keyvalue'), + $container->get('logger.channel.strawberry_runners') + ); + } + + /** + * Get the extractor plugin. + * + * @return object + * The plugin. + * + * @throws \Drupal\Component\Plugin\Exception\PluginException + */ + protected function getProcessorPlugin($plugin_config_entity_id) { + // Get extractor configuration. + /* @var $plugin_config_entity \Drupal\strawberry_runners\Entity\strawberryRunnerPostprocessorEntityInterface */ + $plugin_config_entity = $this->entityTypeManager->getStorage( + 'strawberry_runners_postprocessor' + )->load($plugin_config_entity_id); + + if ($plugin_config_entity->isActive()) { + $entity_id = $plugin_config_entity->id(); + $configuration_options = $plugin_config_entity->getPluginconfig(); + $configuration_options['configEntity'] = $entity_id; + /* @var \Drupal\strawberry_runners\Plugin\StrawberryRunnersPostProcessorPluginInterface $plugin_instance */ + $plugin_instance = $this->strawberryRunnerProcessorPluginManager->createInstance( + $plugin_config_entity->getPluginid(), + $configuration_options + ); + return $plugin_instance; + } + } + + + /** + * {@inheritdoc} + */ + public function processItem($data) { + + // Decode the JSON that was captured. + + $processor_instance = $this->getProcessorPlugin($data->plugin_config_entity_id); + + // Load file from queue item. + $file = $this->entityTypeManager->getStorage('file')->load($data->fid); + + if ($file === NULL) { + return; + } + $filelocation = $this->ensureFileAvailabilty($file); + if ($filelocation === NULL) { + return; + } + + try { + $keyvalue_collection = 'Strawberryfield_flavor_datasource_temp'; + $key = $keyvalue_collection . ':' . $file->uuid().':'.$data->plugin_config_entity_id; + + //We only deal with NODES. + $entity = $this->entityTypeManager->getStorage('node') + ->load($data->nid); + + if(!$entity) { + return; + } + + // Skip file if element is found in key_value collection. + $processed_data = $this->keyValue->get($keyvalue_collection)->get($key); + error_log('processed data in the keyvalue store'); + error_log($processed_data); + if (empty($processed_data)) { + // Extract file and save it in key_value collection. + $io = new \stdClass(); + $input = new \stdClass(); + $input->filepath = $filelocation; + + $io->input = $input; + $io->output = NULL; + $extracted_data = $processor_instance->run($io, StrawberryRunnersPostProcessorPluginInterface::PROCESS); + error_log ('processing just run'); + error_log($io->ouput); + error_log('writing to keyvalue'); + error_log($key); + $this->keyValue->get($keyvalue_collection)->set($key, $io->output); + } + + // Get which indexes have our StrawberryfieldFlavorDatasource enabled! + $indexes = StrawberryfieldFlavorDatasource::getValidIndexes(); + + $item_ids = []; + if (is_a($entity, TranslatableInterface::class)) { + $translations = $entity->getTranslationLanguages(); + foreach ($translations as $translation_id => $translation) { + $item_ids[] = $entity->id() . ':'.'1' .':'.$translation_id.':'.$file->uuid().':'.$data->plugin_config_entity_id; + } + } + error_log(var_export($item_ids,true)); + $datasource_id = 'strawberryfield_flavor_datasource'; + foreach ($indexes as $index) { + $index->trackItemsUpdated($datasource_id, $item_ids); + } + } + catch (\Exception $exception) { + if ($data->extract_attempts < 3) { + $data->extract_attempts++; + \Drupal::queue('strawberryrunners_process_index')->createItem($data); + } + else { + $message_params = [ + '@file_id' => $data->fid, + '@entity_id' => $data->nid, + ]; + $this->logger->log(LogLevel::ERROR, 'Strawberry Runners Processing failed after 3 attempts @file_id for @entity_type @entity_id.', $message_params); + } + } + } + + /** + * Move file to local to if needed process. + * + * @param \Drupal\file\FileInterface $file + * The File URI to look at. + * + * @return array + * Output of processing chain for a particular file. + */ + private function ensureFileAvailabilty(FileInterface $file) { + $uri = $file->getFileUri(); + // Local stream. + $cache_key = md5($uri); + // Check first if the file is already around in temp? + // @TODO can be sure its the same one? Ideas? + if (is_readable( + $this->fileSystem->realpath( + 'temporary://sbr_' . $cache_key . '_' . basename($uri) + ) + )) { + $templocation = $this->fileSystem->realpath( + 'temporary://sbr_' . $cache_key . '_' . basename($uri) + ); + } + else { + $templocation = $this->fileSystem->copy( + $uri, + 'temporary://sbr_' . $cache_key . '_' . basename($uri), + FileSystemInterface::EXISTS_REPLACE + ); + $templocation = $this->fileSystem->realpath( + $templocation + ); + } + + + if (!$templocation) { + $this->loggerFactory->get('strawberry_runners')->warning( + 'Could not adquire a local accessible location for text extraction for file with URL @fileurl', + [ + '@fileurl' => $file->getFileUri(), + ] + ); + return FALSE; + } else { + return $templocation; + } + } + + /** + * Helper method to get the real path from an uri. + * + * @param string $uri + * The URI of the file, e.g. public://directory/file.jpg. + * + * @return mixed + * The real path to the file if it is a local file. An URL otherwise. + */ + public function getRealpath($uri) { + $wrapper = $this->streamWrapperManager->getViaUri($uri); + $scheme = $this->streamWrapperManager->getScheme($uri); + $local_wrappers = $this->streamWrapperManager->getWrappers(StreamWrapperInterface::LOCAL); + if (in_array($scheme, array_keys($local_wrappers))) { + return $wrapper->realpath(); + } + else { + return $wrapper->getExternalUrl(); + } + } + +} \ No newline at end of file diff --git a/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php b/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php index 7749426..aeb8cd3 100644 --- a/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php +++ b/src/Plugin/StrawberryRunnersPostProcessor/SystemBinaryPostProcessor.php @@ -33,7 +33,7 @@ class SystemBinaryPostProcessor extends StrawberryRunnersPostProcessorPluginBase public function defaultConfiguration() { return [ 'source_type' => 'asstructure', - 'mime_type' => 'application/pdf', + 'mime_type' => ['application/pdf'], 'path' => '', 'arguments' => '', 'output_type' => 'json', @@ -153,6 +153,8 @@ public function settingsForm(array $parents, FormStateInterface $form_state) { return $element; } + + public function onDependencyRemoval(array $dependencies) { // Since Processors could be chained we need to check if any other // processor instance is using an instance of this one @@ -174,14 +176,27 @@ public function run(\stdClass $io, $context = StrawberryRunnersPostProcessorPlug // In this case it will contain an absolute Path to a File. // Needed since this executes locally on the server via SHELL. $input_property = $this->pluginDefinition['input_property']; + error_log('run'); + error_log($io->input->{$input_property}); if (isset($io->input->{$input_property})) { + setlocale(LC_CTYPE, 'en_US.UTF-8'); $execstring = $this->buildExecutableCommand($io->input->{$input_property}); + error_log($execstring); if ($execstring) { - $io->output = $execstring; - //dpm($execstring); + $backup_locale = setlocale(LC_CTYPE, '0'); + setlocale(LC_CTYPE, $backup_locale); + // Support UTF-8 commands. + // @see http://www.php.net/manual/en/function.shell-exec.php#85095 + shell_exec("LANG=en_US.utf-8"); + $output = shell_exec($execstring); + if (is_null($output)) { + throw new \Exception("Could not execute {$execstring}"); + } + $io->output = $output; + } } else { - throwException(new \InvalidArgumentException); + \throwException(new \InvalidArgumentException); } } @@ -197,11 +212,17 @@ public function buildExecutableCommand(string $filepath) { $execpath = $config['path']; $arguments = $config['arguments']; $command = ''; - if ($this->verifyCommand($execpath) && (strpos($arguments, '%file' ) !== false)) { + + error_log('verify!'.(int) \Drupal::service('strawberryfield.utility')->verifyCommand($execpath)); + + if (\Drupal::service('strawberryfield.utility')->verifyCommand($execpath) && (strpos($arguments, '%file' ) !== FALSE)) { + error_log('its a command, well well'); $arguments = str_replace('%s','', $arguments); $arguments = str_replace_first('%file','%s', $arguments); $arguments = sprintf($arguments, $filepath); + error_log($arguments); $command = escapeshellcmd($execpath.' '.$arguments); + error_log($command); } // Only return $command if it contains the original filepath somewhere if (strpos($command, $filepath) !== false) { return $command;} diff --git a/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php b/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php index fd3ba06..fb62100 100644 --- a/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php +++ b/src/Plugin/StrawberryRunnersPostProcessorPluginBase.php @@ -75,7 +75,7 @@ public static function create(ContainerInterface $container, array $configuratio public function defaultConfiguration() { return [ 'jsonkey' => ['as:image'], - 'ado_type' => 'Book', + 'ado_type' => ['Book'], // Max time to run in seconds per item. 'timeout' => 10, // Order in which this processor is executed in the chain @@ -108,6 +108,7 @@ public function label() { * {@inheritdoc} */ public function setConfiguration(array $configuration) { + dpm($configuration); $this->configuration = $configuration + $this->defaultConfiguration(); } @@ -133,4 +134,6 @@ public function run(\stdClass $io, $context = strawberryRunnersPostProcessorPlug } + + } \ No newline at end of file diff --git a/src/Plugin/search_api_attachments/SbrWarctotextExtractorRemote.php b/src/Plugin/search_api_attachments/SbrWarctotextExtractorRemote.php new file mode 100644 index 0000000..f737cfc --- /dev/null +++ b/src/Plugin/search_api_attachments/SbrWarctotextExtractorRemote.php @@ -0,0 +1,133 @@ +getMimeType(), ['application/x-gzip'])) { + $output = ''; + $warc_path = $this->configuration['warc_path_external']; + // If the file isn't stored locally make a temporary copy. + $uri = $file->getFileUri(); + $filepath = $this->getRealpath($file->getFileUri()); + + // Local stream. + $cache_key = md5($uri); + // Check first if the file is already around in temp? + // @TODO can be sure its the same one? Ideas? + if (is_readable(\Drupal::service('file_system')->realpath('temporary://sbr_' . $cache_key . '_' . basename($uri)))) { + $templocation = \Drupal::service('file_system')->realpath('temporary://sbr_' . $cache_key . '_' . basename($uri)); + } + else { + $templocation = \Drupal::service('file_system')->copy( + $uri, + 'temporary://sbr_' . $cache_key . '_' . basename($uri), + FileSystemInterface::EXISTS_REPLACE + ); + $templocation = \Drupal::service('file_system')->realpath( + $templocation + ); + } + + + if (!$templocation) { + $this->loggerFactory->get('PdftotextExtractorRemote')->warning( + 'Could not adquire a local accessible location for text extraction for file with URL @fileurl', + [ + '@fileurl' => $file->getFileUri(), + ] + ); + return; + } + + // the default C-locale. + // So temporarily set the locale to UTF-8 so that the filepath remains + // valid. + $backup_locale = setlocale(LC_CTYPE, '0'); + setlocale(LC_CTYPE, 'en_US.UTF-8'); + + $warc_reader = new WarcReader(escapeshellarg($templocation)); + // Using nextRecord, iterate through the WARC file and output each record. + while(($record = $warc_reader->nextRecord()) != FALSE){ + // A WARC record is broken into two parts: header and content. + // header contains metadata about content, while content is the actual resource captured. + dpm($record['header']); + dpm($record['content']); + $output[] = $record['content']; + echo "------------------------------------\n"; + } + $output = implode(';',$output); + //$output = shell_exec($cmd); + if (is_null($output)) { + throw new \Exception('We could not extract.'); + } + return $output; + + } + else { + return NULL; + } + } + + /** + * {@inheritdoc} + */ + public function buildConfigurationForm(array $form, FormStateInterface $form_state) { + $form['warc_path_external'] = [ + '#type' => 'textfield', + '#title' => $this->t('warc-extractor.py binary for external processor (Python 3)'), + '#description' => $this->t('Enter the name of warc-extractor.py python script or the full path. Example: "warc-extractor.py" or "/usr/bin/warc-extractor.py".'), + '#default_value' => $this->configuration['warc_path_external'], + '#required' => TRUE, + ]; + return $form; + } + + /** + * {@inheritdoc} + */ + public function validateConfigurationForm(array &$form, FormStateInterface $form_state) { + $values = $form_state->getValue(['text_extractor_config']); + $warc_path = $values['warc_path_external']; + + $is_name = strpos($warc_path, '/') === FALSE && strpos($warc_path, '\\') === FALSE; + if (!$is_name && !file_exists($warc_path)) { + $form_state->setError($form['text_extractor_config']['warc_path_external'], $this->t('The file %path does not exist.', ['%path' => $warc_path])); + } + } + + /** + * {@inheritdoc} + */ + public function submitConfigurationForm(array &$form, FormStateInterface $form_state) { + $this->configuration['pdftowarc_path_external'] = $form_state->getValue([ + 'text_extractor_config', + 'pdftowarc_path_external', + ]); + parent::submitConfigurationForm($form, $form_state); + } + +} diff --git a/src/StrawberryRunnersLoopService.php b/src/StrawberryRunnersLoopService.php index 4a7ce63..c21792e 100644 --- a/src/StrawberryRunnersLoopService.php +++ b/src/StrawberryRunnersLoopService.php @@ -179,4 +179,5 @@ public function pushItemOnQueue($node_id, $jsondata, $flavour) { + } diff --git a/strawberry_runners.module b/strawberry_runners.module index 1673d31..7638196 100644 --- a/strawberry_runners.module +++ b/strawberry_runners.module @@ -9,3 +9,60 @@ use Drupal\Core\Url; use Drupal\webform\WebformSubmissionForm; use Drupal\Core\Entity\ContentEntityInterface; use Drupal\file\Entity\File; +use Drupal\node\NodeInterface; + + +/** + * Implements hook_node_access_records_alter(). + * + * Marks the Flavor Data Sources as changed for indexes that use the "Content + * access" processor. + */ +function strawberry_runners_node_access_records_alter(array &$grants, NodeInterface $node) { + /** @var \Drupal\search_api\IndexInterface $index */ + return; + foreach (Index::loadMultiple() as $index) { + if (!$index->hasValidTracker() || !$index->status()) { + continue; + } + if (!$index->isValidProcessor('content_access')) { + continue; + } + + foreach ($index->getDatasources() as $datasource_id => $datasource) { + switch ($datasource->getEntityTypeId()) { + case 'node': + // Don't index the node if search_api_skip_tracking is set on it. + if ($node->search_api_skip_tracking) { + continue 2; + } + $item_id = $datasource->getItemId($node->getTypedData()); + if ($item_id !== NULL) { + $index->trackItemsUpdated($datasource_id, [$item_id]); + } + break; + + case 'comment': + if (!isset($comments)) { + $entity_query = \Drupal::entityQuery('comment'); + $entity_query->condition('entity_id', (int) $node->id()); + $entity_query->condition('entity_type', 'node'); + $comment_ids = $entity_query->execute(); + /** @var \Drupal\comment\CommentInterface[] $comments */ + $comments = Comment::loadMultiple($comment_ids); + } + $item_ids = []; + foreach ($comments as $comment) { + $item_id = $datasource->getItemId($comment->getTypedData()); + if ($item_id !== NULL) { + $item_ids[] = $item_id; + } + } + if ($item_ids) { + $index->trackItemsUpdated($datasource_id, $item_ids); + } + break; + } + } + } +} \ No newline at end of file diff --git a/strawberry_runners.services.yml b/strawberry_runners.services.yml index de8d497..33cd8f5 100644 --- a/strawberry_runners.services.yml +++ b/strawberry_runners.services.yml @@ -7,6 +7,11 @@ services: tags: - {name: event_subscriber} arguments: ['@string_translation', '@messenger', '@logger.factory', '@config.factory', '@stream_wrapper_manager', '@file_system', '@entity_type.manager', '@strawberry_runner.processor_manager'] + strawberry_runner.loop: class: Drupal\strawberry_runners\StrawberryRunnersLoopService - arguments: ['@file_system', '@entity_type.manager', '@config.factory', '@module_handler', '@lock', '@queue', '@state', '@account_switcher', '@logger.factory', '@plugin.manager.queue_worker', '@datetime.time'] \ No newline at end of file + arguments: ['@file_system', '@entity_type.manager', '@config.factory', '@module_handler', '@lock', '@queue', '@state', '@account_switcher', '@logger.factory', '@plugin.manager.queue_worker', '@datetime.time'] + + logger.channel.strawberry_runners: + parent: logger.channel_base + arguments: [ 'strawberry_runner' ] \ No newline at end of file