MAGETWO-46245: Multistore: Import product with Replace behaviour caus…

…es an error "URL key for specified store already exists."

app/code/Magento/CatalogUrlRewrite/Model/Category/Plugin/Storage.php

@@ -44,11 +44,13 @@ public function aroundReplace(StorageInterface $object, \Closure $proceed, array
         $toSave = [];
         foreach ($this->filterUrls($urls) as $record) {
             $metadata = $record->getMetadata();
-            $toSave[] = [
-                'url_rewrite_id' => $record->getUrlRewriteId(),
-                'category_id' => $metadata['category_id'],
-                'product_id' => $record->getEntityId(),
-            ];
+            if (isset($metadata['category_id'])) {
+                $toSave[] = [
+                    'url_rewrite_id' => $record->getUrlRewriteId(),
+                    'category_id' => $metadata['category_id'],
+                    'product_id' => $record->getEntityId(),
+                ];
+            }
         }
         if ($toSave) {
             $this->productFactory->create()->getResource()->saveMultiple($toSave);

app/code/Magento/CatalogUrlRewrite/Model/Product/Plugin/Import.php

@@ -158,11 +158,83 @@ public function afterImportData(Observer $observer)
             foreach ($products as $product) {
                 $this->_populateForUrlGeneration($product);
             }
+            /** @var \Magento\UrlRewrite\Service\V1\Data\UrlRewrite[] $productUrls */
             $productUrls = $this->generateUrls();
             if ($productUrls) {
+                try {
+                    $this->urlPersist->replace($productUrls);
+                } catch (\Magento\Framework\Exception\AlreadyExistsException $e) {
+                    $groupsOfUrls = $this->groupUrls($productUrls);
+                    foreach ($groupsOfUrls as $productUrls) {
+                        $this->replaceUrls($productUrls);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Group URL keys by product id and store id
+     *
+     * @param  \Magento\UrlRewrite\Service\V1\Data\UrlRewrite[] $urls
+     * @return array
+     */
+    protected function groupUrls($urls)
+    {
+        $groups =[];
+        foreach ($urls as $url) {
+            $key = sprintf('%s-%s',
+                $url->getEntityId(),
+                $url->getStoreId()
+            );
+            $groups[$key][] = $url;
+        }
+
+        return $groups;
+    }
+
+    /**
+     * Replace product URL by given set
+     *
+     * @param \Magento\UrlRewrite\Service\V1\Data\UrlRewrite[] $productUrls
+     * @throws \Magento\Framework\Exception\AlreadyExistsException
+     * @return $this
+     */
+    protected function replaceUrls(array $productUrls)
+    {
+        for ($i = 0; $i < 100; $i++) {
+            if ($i > 0) {
+                $this->addIncrementToUrls($productUrls, $i);
+            }
+            try {
                 $this->urlPersist->replace($productUrls);
+                break;
+            } catch (\Magento\Framework\Exception\AlreadyExistsException $e) {
+                if ($i == 100) {
+                    throw $e;
+                }
             }
         }
+
+        return $this;
+    }
+
+    /**
+     * Add increment to every URL in array
+     *
+     * @param $productUrls
+     * @param $increment
+     * @return $this
+     */
+    protected function addIncrementToUrls(array & $productUrls, $increment)
+    {
+        foreach($productUrls as $productUrl) {
+            $requestPath = substr($productUrl->getRequestPath(), 0, -strlen($productUrl->getUrlSuffix()));
+            $requestPath = sprintf('%s-%s%s', $requestPath, $increment, $productUrl->getUrlSuffix());
+            $productUrl->setRequestPath($requestPath);
+        }
+
+        return $this;
     }
 
     /**
@@ -310,7 +382,7 @@ protected function cleanOverriddenUrlKey()
      */
     protected function isGlobalScope($storeId)
     {
-        return null === $storeId || $storeId == Store::DEFAULT_STORE_ID;
+        return $storeId === Store::DEFAULT_STORE_ID;
     }
 
     /**
@@ -329,7 +401,8 @@ protected function canonicalUrlRewriteGenerate()
                         ->setEntityId($productId)
                         ->setRequestPath($this->productUrlPathGenerator->getUrlPathWithSuffix($product, $storeId))
                         ->setTargetPath($this->productUrlPathGenerator->getCanonicalUrlPath($product))
-                        ->setStoreId($storeId);
+                        ->setStoreId($storeId)
+                        ->setUrlSuffix($this->productUrlPathGenerator->getUrlSuffix($storeId));
                 }
             }
         }
@@ -356,7 +429,8 @@ protected function categoriesUrlRewriteGenerate()
                         ->setRequestPath($requestPath)
                         ->setTargetPath($this->productUrlPathGenerator->getCanonicalUrlPath($product, $category))
                         ->setStoreId($storeId)
-                        ->setMetadata(['category_id' => $category->getId()]);
+                        ->setMetadata(['category_id' => $category->getId()])
+                        ->setUrlSuffix($this->productUrlPathGenerator->getUrlSuffix($storeId));
                 }
             }
         }

app/code/Magento/CatalogUrlRewrite/Model/ProductUrlPathGenerator.php

@@ -60,9 +60,16 @@ public function getUrlPath($product, $category = null)
     {
         $path = $product->getData('url_path');
         if ($path === null) {
-            $path = $product->getUrlKey() === false
-                ? $this->prepareProductDefaultUrlKey($product)
-                : $this->prepareProductUrlKey($product);
+            if (!$product->getUrlKey()) {
+                try {
+                    $path = $this->prepareProductDefaultUrlKey($product);
+                } catch (\Magento\Framework\Exception\NoSuchEntityException $e) {
+                    // in some cases during import we can face with situation when product does't created yet
+                    $path = $this->prepareProductUrlKey($product);
+                }
+            } else {
+                $path = $this->prepareProductUrlKey($product);
+            }
         }
         return $category === null
             ? $path
@@ -152,4 +159,15 @@ protected function getProductUrlSuffix($storeId = null)
         }
         return $this->productUrlSuffix[$storeId];
     }
+
+    /**
+     * Get suffix thet used for URL generation
+     *
+     * @param null $storeId
+     * @return string
+     */
+    public function getUrlSuffix($storeId = null)
+    {
+        return $this->getProductUrlSuffix($storeId);
+    }
 }

app/code/Magento/CatalogUrlRewrite/Test/Unit/Model/Product/Plugin/ImportTest.php

@@ -446,9 +446,8 @@ public function testAfterImportData()
                 [$this->products[0][ImportProduct::COL_SKU]],
                 [$this->products[1][ImportProduct::COL_SKU]]
             )->willReturn([]);
-        $getProductWebsitesCallsCount = $productsCount * 2;
         $this->importProduct
-            ->expects($this->exactly($getProductWebsitesCallsCount))
+            ->expects($this->exactly(3))
             ->method('getProductWebsites')
            ->willReturnOnConsecutiveCalls(
                [$newSku[0]['entity_id'] => $websiteId],
@@ -495,14 +494,13 @@ public function testAfterImportData()
                $newSku[1]['entity_id']
            );
         $product
-            ->expects($this->exactly($productsCount))
+            ->expects($this->exactly(1))
             ->method('getSku')
             ->will($this->onConsecutiveCalls(
-                $this->products[0]['sku'],
-                $this->products[1]['sku']
+                $this->products[0]['sku']
             ));
         $product
-            ->expects($this->exactly($productsCount))
+            ->expects($this->any($productsCount))
             ->method('getStoreId')
             ->will($this->onConsecutiveCalls(
                 $this->products[0][ImportProduct::COL_STORE],
@@ -520,7 +518,7 @@ public function testAfterImportData()
             ->method('create')
             ->willReturn($product);
         $this->connection
-            ->expects($this->exactly(4))
+            ->expects($this->exactly(2))
             ->method('quoteInto')
             ->withConsecutive(
                 [
@@ -562,8 +560,9 @@ public function testAfterImportData()
         $this->urlRewrite->expects($this->any())->method('setRequestPath')->willReturnSelf();
         $this->urlRewrite->expects($this->any())->method('setTargetPath')->willReturnSelf();
         $this->urlRewrite->expects($this->any())->method('getTargetPath')->willReturn('targetPath');
+        $this->urlRewrite->expects($this->any())->method('setUrlSuffix')->willReturnSelf();
         $this->urlRewrite->expects($this->any())->method('getStoreId')
-            ->willReturnOnConsecutiveCalls(0, 'not global');
+            ->willReturnOnConsecutiveCalls(0, 'not global', 0, 'not global');
 
         $this->urlRewriteFactory->expects($this->any())->method('create')->willReturn($this->urlRewrite);
 
@@ -685,13 +684,17 @@ public function testCanonicalUrlRewriteGenerateWithUrlPath()
             ->method('getUrlPathWithSuffix')
             ->will($this->returnValue($requestPath));
         $this->productUrlPathGenerator
-            ->expects($this->once())
+            ->expects($this->any())
             ->method('getUrlPath')
             ->will($this->returnValue('urlPath'));
         $this->productUrlPathGenerator
             ->expects($this->once())
             ->method('getCanonicalUrlPath')
             ->will($this->returnValue($targetPath));
+        $this->productUrlPathGenerator
+            ->expects($this->once())
+            ->method('getUrlSuffix')
+            ->will($this->returnValue('.html'));
         $this->urlRewrite
             ->expects($this->once())
             ->method('setStoreId')
@@ -717,6 +720,16 @@ public function testCanonicalUrlRewriteGenerateWithUrlPath()
             ->method('setTargetPath')
             ->with($targetPath)
             ->will($this->returnSelf());
+        $this->urlRewrite
+            ->expects($this->once())
+            ->method('setStoreId')
+            ->with(10)
+            ->will($this->returnSelf());
+        $this->urlRewrite
+            ->expects($this->once())
+            ->method('setUrlSuffix')
+            ->with('.html')
+            ->will($this->returnSelf());
         $this->urlRewriteFactory
             ->expects($this->once())
             ->method('create')
@@ -796,6 +809,10 @@ public function testCategoriesUrlRewriteGenerate()
             ->expects($this->any())
             ->method('getCanonicalUrlPath')
             ->will($this->returnValue($canonicalUrlPathWithCategory));
+        $this->productUrlPathGenerator
+            ->expects($this->any())
+            ->method('getUrlSuffix')
+            ->will($this->returnValue('.html'));
         $category = $this->getMock('Magento\Catalog\Model\Category', [], [], '', false);
         $category
             ->expects($this->any())
@@ -831,6 +848,11 @@ public function testCategoriesUrlRewriteGenerate()
             ->method('setMetadata')
             ->with(['category_id' => $this->categoryId])
             ->will($this->returnSelf());
+        $this->urlRewrite
+            ->expects($this->any())
+            ->method('setUrlSuffix')
+            ->with('.html')
+            ->will($this->returnSelf());
         $this->urlRewriteFactory
             ->expects($this->any())
             ->method('create')

app/code/Magento/UrlRewrite/Service/V1/Data/UrlRewrite.php

@@ -37,6 +37,12 @@ class UrlRewrite extends AbstractSimpleObject
         self::DESCRIPTION => null,
     ];
 
+    /**
+     * URL suffix for rewrite (used in import)
+     * @var
+     */
+    protected $urlSuffix;
+
     /**
      * Get data by key
      *
@@ -232,6 +238,27 @@ public function setMetadata($metadata)
         return $this->setData(UrlRewrite::METADATA, $metadata);
     }
 
+    /**
+     * Store url suffix
+     *
+     * @param string $suffix
+     * @return $this
+     */
+    public function setUrlSuffix($suffix)
+    {
+        $this->urlSuffix = (string) $suffix;
+        return $this;
+    }
+
+    /**
+     * Return URL suffix in case if it exists
+     * @return string
+     */
+    public function getUrlSuffix()
+    {
+        return $this->urlSuffix;
+    }
+
     /**
      * Convert UrlRewrite to array
      *