fix(xo-vmdk-to-vhd): better computation of overprovisioning of very sparse disks

CHANGELOG.unreleased.md

@@ -11,6 +11,8 @@
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
+- [Ova export] Better computation of overprovisioning for very sparse disks (PR [#6639](https://github.com/vatesfr/xen-orchestra/pull/6639))
+
 ### Packages to release
 
 > When modifying a package, add it here with its release type.
@@ -27,6 +29,7 @@
 
 <!--packages-start-->
 
+- xo-vmdk-to-vhd patch
 - xo-server-upload-ova patch
 
 <!--packages-end-->

packages/xo-vmdk-to-vhd/src/index.js

@@ -67,12 +67,11 @@ export async function vhdToVMDK(diskName, vhdReadStreamGetter, withLength = fals
 export async function vhdToVMDKIterator(diskName, vhdReadStream) {
   const { blockSize, blockCount, blocks, diskSize, geometry } = await parseVhdToBlocks(vhdReadStream)
 
-  const vmdkTargetSize = blockSize * blockCount + 3 * 1024 * 1024 // header/footer/descriptor
-  const iterator = await generateVmdkData(diskName, diskSize, blockSize, blocks, geometry, vmdkTargetSize)
-
+  const dataSize = blockSize * blockCount
+  const { iterator, metadataSize } = await generateVmdkData(diskName, diskSize, blockSize, blocks, geometry, dataSize)
   return {
     iterator,
-    size: vmdkTargetSize,
+    size: dataSize + metadataSize,
   }
 }
 

packages/xo-vmdk-to-vhd/src/vmdk-generate.js

@@ -11,6 +11,8 @@ import {
   MARKER_EOS,
 } from './definitions'
 
+const roundToSector = value => Math.ceil(value / SECTOR_SIZE) * SECTOR_SIZE
+
 /**
  * - block is an input bunch of bytes, VHD default size is 2MB
  * - grain is an output (VMDK) bunch of bytes, VMDK default is 64KB
@@ -34,7 +36,7 @@ export async function generateVmdkData(
     heads: 16,
     cylinders: 10402,
   },
-  targetSize
+  dataSize
 ) {
   const cid = Math.floor(Math.random() * Math.pow(2, 32))
   const diskCapacitySectors = Math.ceil(diskCapacityBytes / SECTOR_SIZE)
@@ -81,8 +83,8 @@ ddb.geometry.cylinders = "${geometry.cylinders}"
 
   let streamPosition = 0
   let directoryOffset = 0
-
-  const roundToSector = value => Math.ceil(value / SECTOR_SIZE) * SECTOR_SIZE
+  const endMetadataLength = computeEndMetadataLength()
+  const metadataSize = headerData.buffer.length + descriptorBuffer.length + endMetadataLength
 
   function track(buffer) {
     assert.equal(streamPosition % SECTOR_SIZE, 0)
@@ -151,25 +153,31 @@ ddb.geometry.cylinders = "${geometry.cylinders}"
     }
   }
 
+  function computeEndMetadataLength() {
+    return (
+      SECTOR_SIZE + // MARKER_GT
+      roundToSector(tableBuffer.length) +
+      SECTOR_SIZE + // MARKER_GD
+      roundToSector(headerData.grainDirectoryEntries * 4) +
+      SECTOR_SIZE + // MARKER_GT
+      roundToSector(tableBuffer.length) +
+      SECTOR_SIZE + // MARKER_GD
+      roundToSector(headerData.grainDirectoryEntries * 4) +
+      SECTOR_SIZE + // MARKER_FOOTER
+      SECTOR_SIZE + // stream optimizedheader
+      SECTOR_SIZE // MARKER_EOS
+    )
+  }
+
   function* padding() {
-    if (targetSize === undefined) {
+    if (dataSize === undefined) {
       return
     }
-    let remaining = targetSize - streamPosition
-    remaining -= SECTOR_SIZE // MARKER_GT
-    remaining -= tableBuffer.length
-    remaining -= SECTOR_SIZE // MARKER_GD
-    remaining -= roundToSector(headerData.grainDirectoryEntries * 4)
-    remaining -= SECTOR_SIZE // MARKER_GT
-    remaining -= tableBuffer.length
-    remaining -= SECTOR_SIZE // MARKER_GD
-    remaining -= roundToSector(headerData.grainDirectoryEntries * 4)
-    remaining -= SECTOR_SIZE // MARKER_FOOTER
-    remaining -= SECTOR_SIZE // stream optimizedheader
-    remaining -= SECTOR_SIZE // MARKER_EOS
+    const targetSize = dataSize + metadataSize
+    let remaining = targetSize - streamPosition - endMetadataLength
 
     if (remaining < 0) {
-      throw new Error('vmdk is bigger than precalculed size ')
+      throw new Error(`vmdk is bigger than precalculed size`)
     }
     const size = 1024 * 1024
     const fullBuffer = Buffer.alloc(size, 0)
@@ -212,5 +220,8 @@ ddb.geometry.cylinders = "${geometry.cylinders}"
     yield track(footer.buffer)
     yield track(createEmptyMarker(MARKER_EOS))
   }
-  return iterator()
+  return {
+    iterator: iterator(),
+    metadataSize,
+  }
 }

packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js

@@ -85,7 +85,8 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async (
 })
 
 test('Can generate an empty VMDK file', async () => {
-  const readStream = asyncIteratorToStream(await generateVmdkData('result.vmdk', 1024 * 1024 * 1024, 1024 * 1024, []))
+  const { iterator } = await generateVmdkData('result.vmdk', 1024 * 1024 * 1024, 1024 * 1024, [])
+  const readStream = asyncIteratorToStream(iterator)
   const pipe = readStream.pipe(createWriteStream('result.vmdk'))
   await fromEvent(pipe, 'finish')
   await execa('qemu-img', ['check', 'result.vmdk'])
@@ -102,9 +103,8 @@ test('Can generate a small VMDK file', async () => {
   ]
   const fileName = 'result.vmdk'
   const geometry = { sectorsPerTrackCylinder: 63, heads: 16, cylinders: 10402 }
-  const readStream = asyncIteratorToStream(
-    await generateVmdkData(fileName, 2 * blockSize, blockSize, blockGenerator, geometry)
-  )
+  const { iterator } = await await generateVmdkData(fileName, 2 * blockSize, blockSize, blockGenerator, geometry)
+  const readStream = asyncIteratorToStream(iterator)
   const pipe = readStream.pipe(createWriteStream(fileName))
   await fromEvent(pipe, 'finish')