Feature/1078 asset indexing improvements

CHANGELOG-WIP.md

@@ -35,6 +35,10 @@
 - The `plugin/install`, `plugin/uninstall`, `plugin/enable`, and `plugin/disabled` commands now support an `--all` option, which applies the action to all applicable Composer-installed plugins. ([#11373](https://github.com/craftcms/cms/discussions/11373), [#12218](https://github.com/craftcms/cms/pull/12218))
 - The `project-config/apply` command now supports a `--quiet` option, which reduces the command output. ([#12568](https://github.com/craftcms/cms/discussions/12568))
 - Added the `users/unlock` console command. ([#12345](https://github.com/craftcms/cms/discussions/12345))
+- The Asset Indexes utility no longer skips volumes if the root folder was completely empty. ([#12585](https://github.com/craftcms/cms/issues/12585), [#12604](https://github.com/craftcms/cms/pull/12604))
+- The Asset Indexes utility now has a “List empty folders” setting, which determines whether empty folders sholud be listed for deletion from the index. ([#12604](https://github.com/craftcms/cms/pull/12604))
+- The Asset Indexes utility now lists missing/empty folders and files separately in the review screen. ([#12604](https://github.com/craftcms/cms/pull/12604))
+- Improved the CLI output for `index-assets` commands. ([#12604](https://github.com/craftcms/cms/pull/12604))
 
 ### Development
 - Added the `exec` command, which executes an individual PHP statement and outputs the result. ([#12528](https://github.com/craftcms/cms/pull/12528))
@@ -97,6 +101,7 @@
 - Added `craft\utilities\AssetIndexes::EVENT_LIST_VOLUMES`. ([#12383](https://github.com/craftcms/cms/pull/12383), [#12443](https://github.com/craftcms/cms/pull/12443))
 - Renamed `craft\elements\conditions\assets\EditableConditionRule` to `SavableConditionRule`, while preserving the original class name with an alias. ([#12266](https://github.com/craftcms/cms/pull/12266))
 - Renamed `craft\elements\conditions\entries\EditableConditionRule` to `SavableConditionRule`, while preserving the original class name with an alias. ([#12266](https://github.com/craftcms/cms/pull/12266))
+- `craft\services\AssetIndexer::startIndexingSession()` and `createIndexingSession()` now have a `$listEmptyFolders` argument. ([#12604](https://github.com/craftcms/cms/pull/12604))
 - `craft\base\ElementQuery::joinElementTable()` now accepts table names in the format of `{{%tablename}}`.
 - Deprecated `craft\imagetransforms\ImageTransformer::ensureTransformUrlByIndexModel()`. `getTransformUrl()` should be used instead.
 - Deprecated `craft\imagetransforms\ImageTransformer::procureTransformedImage()`. `generateTransform()` should be used instead.

src/config/app.php

@@ -4,7 +4,7 @@
     'id' => 'CraftCMS',
     'name' => 'Craft CMS',
     'version' => '4.3.6.1',
-    'schemaVersion' => '4.4.0.1',
+    'schemaVersion' => '4.4.0.2',
     'minVersionRequired' => '3.7.11',
     'basePath' => dirname(__DIR__), // Defines the @app alias
     'runtimePath' => '@storage/runtime', // Defines the @runtime alias

src/console/controllers/IndexAssetsController.php

@@ -235,6 +235,16 @@ private function _indexAssets(array $volumes, string $path = '', int $startAt =
             $this->stdout(PHP_EOL);
         }
 
+        if (!empty($missingFolders)) {
+            $totalMissing = count($missingFolders);
+            $this->stdout(($totalMissing === 1 ? 'One missing folder:' : "$totalMissing missing folders:") . PHP_EOL, Console::FG_YELLOW);
+            foreach ($missingFolders as $folderId => $folderPath) {
+                $this->stdout("- $folderPath ($folderId)");
+                $this->stdout(PHP_EOL);
+            }
+            $this->stdout(PHP_EOL);
+        }
+
         $remainingMissingFiles = $missingFiles;
 
         if ($maybes && $this->interactive && $this->confirm('Fix asset locations?')) {

src/controllers/AssetIndexesController.php

@@ -52,20 +52,21 @@ public function actionStartIndexing(): Response
         $request = Craft::$app->getRequest();
         $volumes = (array)$request->getRequiredBodyParam('volumes');
         $cacheRemoteImages = (bool)$request->getBodyParam('cacheImages', false);
+        $listEmptyFolders = (bool)$request->getBodyParam('listEmptyFolders', false);
 
         if (empty($volumes)) {
             return $this->asFailure(Craft::t('app', 'No volumes specified.'));
         }
 
-        $indexingSession = Craft::$app->getAssetIndexer()->startIndexingSession($volumes, $cacheRemoteImages);
+        $indexingSession = Craft::$app->getAssetIndexer()->startIndexingSession($volumes, $cacheRemoteImages, $listEmptyFolders);
         $sessionData = $this->prepareSessionData($indexingSession);
 
         $data = ['session' => $sessionData];
         $error = null;
 
-        if ($indexingSession->totalEntries === 0) {
+        if ($indexingSession->totalEntries === 0 && !$indexingSession->processIfRootEmpty) {
             $data['stop'] = $indexingSession->id;
-            $error = Craft::t('app', 'Nothing to index.');
+            $error = Craft::t('app', 'The filesystem doesn’t contain any files.');
             Craft::$app->getAssetIndexer()->stopIndexingSession($indexingSession);
         }
 

src/migrations/Install.php

@@ -146,8 +146,10 @@ public function createTables(): void
             'totalEntries' => $this->integer(),
             'processedEntries' => $this->integer()->notNull()->defaultValue(0),
             'cacheRemoteImages' => $this->boolean(),
+            'listEmptyFolders' => $this->boolean()->defaultValue(false),
             'isCli' => $this->boolean()->defaultValue(false),
             'actionRequired' => $this->boolean()->defaultValue(false),
+            'processIfRootEmpty' => $this->boolean()->defaultValue(false),
             'dateCreated' => $this->dateTime()->notNull(),
             'dateUpdated' => $this->dateTime()->notNull(),
             'uid' => $this->uid(),

src/migrations/m230131_120713_asset_indexing_session_new_options.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace craft\migrations;
+
+use craft\db\Migration;
+use craft\db\Table;
+
+/**
+ * m230131_120713_asset_indexing_session_new_options migration.
+ */
+class m230131_120713_asset_indexing_session_new_options extends Migration
+{
+    /**
+     * @inheritdoc
+     */
+    public function safeUp(): bool
+    {
+        $table = $this->db->schema->getTableSchema(Table::ASSETINDEXINGSESSIONS);
+
+        if (!isset($table->columns['processIfRootEmpty'])) {
+            $this->addColumn(
+                Table::ASSETINDEXINGSESSIONS,
+                'processIfRootEmpty',
+                $this->boolean()->defaultValue(false)->after('actionRequired'),
+            );
+        }
+        if (!isset($table->columns['listEmptyFolders'])) {
+            $this->addColumn(
+                Table::ASSETINDEXINGSESSIONS,
+                'listEmptyFolders',
+                $this->boolean()->defaultValue(false)->after('cacheRemoteImages'),
+            );
+        }
+
+        return true;
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function safeDown(): bool
+    {
+        $table = $this->db->schema->getTableSchema(Table::ASSETINDEXINGSESSIONS);
+        if (isset($table->columns['processIfRootEmpty'])) {
+            $this->dropColumn(Table::ASSETINDEXINGSESSIONS, 'processIfRootEmpty');
+        }
+        if (isset($table->columns['listEmptyFolders'])) {
+            $this->dropColumn(Table::ASSETINDEXINGSESSIONS, 'listEmptyFolders');
+        }
+        return true;
+    }
+}

src/models/AssetIndexingSession.php

@@ -43,6 +43,12 @@ class AssetIndexingSession extends Model
      */
     public bool $cacheRemoteImages;
 
+    /**
+     * @var bool Whether empty folders should be listed for deletion.
+     * @since 4.4.0
+     */
+    public bool $listEmptyFolders;
+
     /**
      * Whether this session runs in CLI.
      *
@@ -74,4 +80,10 @@ class AssetIndexingSession extends Model
      * @var array The missing entries.
      */
     public array $missingEntries = [];
+
+    /**
+     * @var bool Whether to continue processing if the FS root folder is empty.
+     * @since 4.4.0
+     */
+    public bool $processIfRootEmpty = false;
 }

src/records/AssetIndexingSession.php

@@ -18,8 +18,10 @@
  * @property int|null $totalEntries The total amount of entries.
  * @property int|null $processedEntries The number of processed entries.
  * @property bool $cacheRemoteImages Whether remote images should be cached locally.
+ * @property bool $listEmptyFolders Whether to list empty folders for deletion.
  * @property bool $isCli Whether indexing is run via CLI.
  * @property bool $actionRequired Whether action is required.
+ * @property bool $processIfRootEmpty Whether to continue processing if the FS root folder is empty.
  * @property string $dateUpdated Time when indexing session was last updated.
  * @property string $dateCreated Time when indexing session was last updated.
  *

src/services/AssetIndexer.php

@@ -26,6 +26,7 @@
 use craft\helpers\Image;
 use craft\helpers\ImageTransforms;
 use craft\helpers\Json;
+use craft\helpers\StringHelper;
 use craft\models\AssetIndexData;
 use craft\models\AssetIndexingSession;
 use craft\models\FsListing;
@@ -142,10 +143,11 @@ public function getIndexingSessionById(int $sessionId): ?AssetIndexingSession
      *
      * @param array $volumes
      * @param bool $cacheRemoteImages
+     * @param bool $listEmptyFolders
      * @return AssetIndexingSession
      * @since 4.0.0
      */
-    public function startIndexingSession(array $volumes, bool $cacheRemoteImages = true): AssetIndexingSession
+    public function startIndexingSession(array $volumes, bool $cacheRemoteImages = true, bool $listEmptyFolders = false): AssetIndexingSession
     {
         $volumeList = [];
         $volumeService = Craft::$app->getVolumes();
@@ -160,7 +162,7 @@ public function startIndexingSession(array $volumes, bool $cacheRemoteImages = t
             }
         }
 
-        $session = $this->createIndexingSession($volumeList, $cacheRemoteImages);
+        $session = $this->createIndexingSession($volumeList, $cacheRemoteImages, listEmptyFolders: $listEmptyFolders);
         $total = 0;
 
         /** @var Volume $volume */
@@ -169,6 +171,9 @@ public function startIndexingSession(array $volumes, bool $cacheRemoteImages = t
             $total += $this->storeIndexList($fileList, $session->id, (int)$volume->id);
         }
 
+        if ($total === 0) {
+            $session->processIfRootEmpty = true;
+        }
         $session->totalEntries = $total;
         $this->storeIndexingSession($session);
 
@@ -193,10 +198,11 @@ public function stopIndexingSession(AssetIndexingSession $session): void
      * @param Volume[] $volumeList
      * @param bool $cacheRemoteImages Whether remote images should be cached.
      * @param bool $isCli Whether indexing is run via CLI
+     * @param bool $listEmptyFolders Whether empty folders should be listed for deletion.
      * @return AssetIndexingSession
      * @since 4.0.0
      */
-    public function createIndexingSession(array $volumeList, bool $cacheRemoteImages = true, bool $isCli = false): AssetIndexingSession
+    public function createIndexingSession(array $volumeList, bool $cacheRemoteImages = true, bool $isCli = false, bool $listEmptyFolders = false): AssetIndexingSession
     {
         $indexedVolumes = [];
 
@@ -209,9 +215,11 @@ public function createIndexingSession(array $volumeList, bool $cacheRemoteImages
             'indexedVolumes' => Json::encode($indexedVolumes),
             'processedEntries' => 0,
             'cacheRemoteImages' => $cacheRemoteImages,
+            'listEmptyFolders' => $listEmptyFolders,
             'actionRequired' => false,
             'isCli' => $isCli,
             'dateUpdated' => null,
+            'processIfRootEmpty' => false,
         ]);
 
         $this->storeIndexingSession($session);
@@ -236,8 +244,10 @@ protected function storeIndexingSession(AssetIndexingSession $session): void
         $record->totalEntries = $session->totalEntries;
         $record->processedEntries = $session->processedEntries;
         $record->cacheRemoteImages = $session->cacheRemoteImages;
+        $record->listEmptyFolders = $session->listEmptyFolders;
         $record->actionRequired = $session->actionRequired;
         $record->isCli = $session->isCli;
+        $record->processIfRootEmpty = $session->processIfRootEmpty;
         $record->save();
 
         $session->id = $record->id;
@@ -295,34 +305,41 @@ public function processIndexSession(AssetIndexingSession $indexingSession): Asse
         $indexEntry = $this->getNextIndexEntry($indexingSession);
 
         // The most likely scenario is that the last entry is being worked on.
-        if (!$indexEntry) {
+        if (!$indexEntry && !$indexingSession->processIfRootEmpty) {
             $mutex->release($lockName);
             return $indexingSession;
         }
 
         // Mark as started.
-        $this->updateIndexEntry($indexEntry->id, ['inProgress' => true]);
-        $mutex->release($lockName);
+        if ($indexEntry) {
+            $this->updateIndexEntry($indexEntry->id, ['inProgress' => true]);
+            $mutex->release($lockName);
 
-        try {
-            if ($indexEntry->isDir) {
-                $recordId = $this->indexFolderByEntry($indexEntry)->id;
-            } else {
-                $recordId = $this->indexFileByEntry($indexEntry, $indexingSession->cacheRemoteImages)->id;
+            try {
+                if ($indexEntry->isDir) {
+                    $recordId = $this->indexFolderByEntry($indexEntry)->id;
+                } else {
+                    $recordId = $this->indexFileByEntry($indexEntry, $indexingSession->cacheRemoteImages)->id;
+                }
+
+                $this->updateIndexEntry($indexEntry->id, ['completed' => true, 'inProgress' => false, 'recordId' => $recordId]);
+            } catch (AssetDisallowedExtensionException|AssetNotIndexableException) {
+                $this->updateIndexEntry($indexEntry->id, ['completed' => true, 'inProgress' => false, 'isSkipped' => true]);
+            } catch (Throwable $exception) {
+                Craft::$app->getErrorHandler()->logException($exception);
+                $this->updateIndexEntry($indexEntry->id, ['completed' => true, 'inProgress' => false, 'isSkipped' => true]);
             }
 
-            $this->updateIndexEntry($indexEntry->id, ['completed' => true, 'inProgress' => false, 'recordId' => $recordId]);
-        } catch (AssetDisallowedExtensionException|AssetNotIndexableException) {
-            $this->updateIndexEntry($indexEntry->id, ['completed' => true, 'inProgress' => false, 'isSkipped' => true]);
-        } catch (Throwable $exception) {
-            Craft::$app->getErrorHandler()->logException($exception);
-            $this->updateIndexEntry($indexEntry->id, ['completed' => true, 'inProgress' => false, 'isSkipped' => true]);
+            $session = $this->incrementProcessedEntryCount($indexingSession);
+        } else {
+            $session = $indexingSession;
         }
 
-        $session = $this->incrementProcessedEntryCount($indexingSession);
-
         if ($session->processedEntries == $session->totalEntries) {
             $session->actionRequired = true;
+            if ($session->processIfRootEmpty) {
+                $session->processIfRootEmpty = false;
+            }
             $this->storeIndexingSession($session);
         }
 
@@ -384,16 +401,21 @@ public function getMissingEntriesForSession(AssetIndexingSession $session): arra
 
         $volumeList = array_keys($volumeList);
 
-        $missingFolders = (new Query())
+        $missingFoldersQuery = (new Query())
             ->select(['path' => 'folders.path', 'volumeName' => 'volumes.name', 'volumeId' => 'volumes.id', 'folderId' => 'folders.id'])
             ->from(['folders' => Table::VOLUMEFOLDERS])
             ->leftJoin(['volumes' => Table::VOLUMES], '[[volumes.id]] = [[folders.volumeId]]')
-            ->leftJoin(['indexData' => Table::ASSETINDEXDATA], ['and', '[[folders.id]] = [[indexData.recordId]]', ['indexData.isDir' => true]])
             ->where(['<', 'folders.dateCreated', $cutoff])
             ->andWhere(['folders.volumeId' => $volumeList])
-            ->andWhere(['not', ['folders.parentId' => null]])
-            ->andWhere(['indexData.id' => null])
-            ->all();
+            ->andWhere(['not', ['folders.parentId' => null]]);
+
+        if (!$session->listEmptyFolders) {
+            $missingFoldersQuery
+                ->leftJoin(['indexData' => Table::ASSETINDEXDATA], ['and', '[[folders.id]] = [[indexData.recordId]]', ['indexData.isDir' => true]])
+                ->andWhere(['indexData.id' => null]);
+        }
+
+        $missingFolders = $missingFoldersQuery->all();
 
         $missingFiles = (new Query())
             ->select(['path' => 'folders.path', 'volumeName' => 'volumes.name', 'filename' => 'assets.filename', 'assetId' => 'assets.id'])
@@ -416,13 +438,25 @@ public function getMissingEntriesForSession(AssetIndexingSession $session): arra
             $hasAssets = (new Query())
                 ->from(['a' => Table::ASSETS])
                 ->innerJoin(['f' => Table::VOLUMEFOLDERS], '[[f.id]] = [[a.folderId]]')
+                ->leftJoin(['e' => Table::ELEMENTS], '[[e.id]] = [[a.id]]')
                 ->where(['a.volumeId' => $volumeId])
                 ->andWhere(['like', 'f.path', "$path%", false])
-                ->exists();
+                ->andWhere(['e.dateDeleted' => null])
+                ->count();
 
-            if (!$hasAssets) {
+            if ($hasAssets == 0) {
                 $missing['folders'][$folderId] = $volumeName . '/' . $path;
             }
+
+            if ($session->listEmptyFolders && $hasAssets > 0) {
+                // if the folder contains as many assets as are listed in the $missingFiles
+                // allow this folder to be offered for deletion (with the assets in it)
+                if ($hasAssets == count(array_filter($missingFiles, function($file) use ($path) {
+                    return StringHelper::startsWith($file['path'], $path);
+                }))) {
+                    $missing['folders'][$folderId] = $volumeName . '/' . $path;
+                }
+            }
         }
 
         foreach ($missingFiles as ['assetId' => $assetId, 'path' => $path, 'volumeName' => $volumeName, 'filename' => $filename]) {
@@ -827,8 +861,10 @@ private function _createAssetIndexingSessionQuery(): Query
                 'totalEntries',
                 'processedEntries',
                 'cacheRemoteImages',
+                'listEmptyFolders',
                 'isCli',
                 'actionRequired',
+                'processIfRootEmpty',
                 'dateCreated',
                 'dateUpdated',
             ])

src/templates/_components/utilities/AssetIndexes.twig

@@ -31,6 +31,14 @@
         on: true,
     }) }}
 
+    {{ forms.lightswitchField({
+        name: 'listEmptyFolders',
+        label: 'List empty folders'|t('app'),
+        class: 'volume-selector',
+        instructions: 'Whether empty folders should be listed for deletion.'|t('app'),
+        on: false,
+    }) }}
+
     <div class="buttons">
         <button type="submit" class="btn submit">{{ 'Update asset indexes'|t('app') }}</button>
         <div class="utility-status"></div>
@@ -49,6 +57,7 @@
             totalEntries: {{ session.totalEntries }},
             processedEntries: {{ session.processedEntries }},
             actionRequired: {{ session.actionRequired + 0 }},
+            processIfRootEmpty: {{ session.processIfRootEmpty + 0 }},
             skippedEntries: [],
             missingEntries: [],
             dateCreated: "{{ session.dateCreated|date(dateFormat) }}",
@@ -72,6 +81,7 @@
 
         const data = {
             'cacheImages': !!$('input[name=cacheImages]').val(),
+            'listEmptyFolders': !!$('input[name=listEmptyFolders]').val(),
             'volumes': $(ev.target).find('.checkbox-select input:checked:enabled').map(function(){return $(this).val();}).get()
         };