From 04eeb5029d3dac56b90fa9323f059f796e76f490 Mon Sep 17 00:00:00 2001 From: florent Beauchamp Date: Wed, 25 Jan 2023 15:46:23 +0100 Subject: [PATCH 1/5] Fix(xo-vmdk-to-vhd): over overprovisionningof very sparse disks --- packages/xo-vmdk-to-vhd/src/index.js | 7 ++- packages/xo-vmdk-to-vhd/src/vmdk-generate.js | 47 ++++++++++++------- .../src/vmdk-to-vhd.integ.spec.js | 3 +- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/packages/xo-vmdk-to-vhd/src/index.js b/packages/xo-vmdk-to-vhd/src/index.js index a873a91ea3f..7d40036d870 100644 --- a/packages/xo-vmdk-to-vhd/src/index.js +++ b/packages/xo-vmdk-to-vhd/src/index.js @@ -67,12 +67,11 @@ export async function vhdToVMDK(diskName, vhdReadStreamGetter, withLength = fals export async function vhdToVMDKIterator(diskName, vhdReadStream) { const { blockSize, blockCount, blocks, diskSize, geometry } = await parseVhdToBlocks(vhdReadStream) - const vmdkTargetSize = blockSize * blockCount + 3 * 1024 * 1024 // header/footer/descriptor - const iterator = await generateVmdkData(diskName, diskSize, blockSize, blocks, geometry, vmdkTargetSize) - + const dataSize = blockSize * blockCount + const { iterator, metadataSize } = await generateVmdkData(diskName, diskSize, blockSize, blocks, geometry, dataSize) return { iterator, - size: vmdkTargetSize, + size: dataSize + metadataSize, } } diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-generate.js b/packages/xo-vmdk-to-vhd/src/vmdk-generate.js index e426bb728ef..694446efa68 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-generate.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-generate.js @@ -11,6 +11,8 @@ import { MARKER_EOS, } from './definitions' +const roundToSector = value => Math.ceil(value / SECTOR_SIZE) * SECTOR_SIZE + /** * - block is an input bunch of bytes, VHD default size is 2MB * - grain is an output (VMDK) bunch of bytes, VMDK default is 64KB @@ -34,7 +36,7 @@ export async function generateVmdkData( heads: 16, cylinders: 10402, }, - targetSize + dataSize ) { const cid = Math.floor(Math.random() * Math.pow(2, 32)) const diskCapacitySectors = Math.ceil(diskCapacityBytes / SECTOR_SIZE) @@ -81,8 +83,8 @@ ddb.geometry.cylinders = "${geometry.cylinders}" let streamPosition = 0 let directoryOffset = 0 - - const roundToSector = value => Math.ceil(value / SECTOR_SIZE) * SECTOR_SIZE + const endMetadataLength = computeEndMetadataLength() + const metadataSize = headerData.buffer.length + descriptorBuffer.length + endMetadataLength function track(buffer) { assert.equal(streamPosition % SECTOR_SIZE, 0) @@ -151,25 +153,31 @@ ddb.geometry.cylinders = "${geometry.cylinders}" } } + function computeEndMetadataLength() { + return ( + SECTOR_SIZE + // MARKER_GT + roundToSector(tableBuffer.length) + + SECTOR_SIZE + // MARKER_GD + roundToSector(headerData.grainDirectoryEntries * 4) + + SECTOR_SIZE + // MARKER_GT + roundToSector(tableBuffer.length) + + SECTOR_SIZE + // MARKER_GD + roundToSector(headerData.grainDirectoryEntries * 4) + + SECTOR_SIZE + // MARKER_FOOTER + SECTOR_SIZE + // stream optimizedheader + SECTOR_SIZE + ) // MARKER_EOS + } + function* padding() { - if (targetSize === undefined) { + if (dataSize === undefined) { return } - let remaining = targetSize - streamPosition - remaining -= SECTOR_SIZE // MARKER_GT - remaining -= tableBuffer.length - remaining -= SECTOR_SIZE // MARKER_GD - remaining -= roundToSector(headerData.grainDirectoryEntries * 4) - remaining -= SECTOR_SIZE // MARKER_GT - remaining -= tableBuffer.length - remaining -= SECTOR_SIZE // MARKER_GD - remaining -= roundToSector(headerData.grainDirectoryEntries * 4) - remaining -= SECTOR_SIZE // MARKER_FOOTER - remaining -= SECTOR_SIZE // stream optimizedheader - remaining -= SECTOR_SIZE // MARKER_EOS + const targetSize = dataSize + metadataSize + let remaining = targetSize - streamPosition - endMetadataLength if (remaining < 0) { - throw new Error('vmdk is bigger than precalculed size ') + throw new Error(`vmdk is bigger than precalculed size`) } const size = 1024 * 1024 const fullBuffer = Buffer.alloc(size, 0) @@ -212,5 +220,8 @@ ddb.geometry.cylinders = "${geometry.cylinders}" yield track(footer.buffer) yield track(createEmptyMarker(MARKER_EOS)) } - return iterator() + return { + iterator: iterator(), + metadataSize, + } } diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js b/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js index ecf4551feea..19a7e375501 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js @@ -85,7 +85,8 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async ( }) test('Can generate an empty VMDK file', async () => { - const readStream = asyncIteratorToStream(await generateVmdkData('result.vmdk', 1024 * 1024 * 1024, 1024 * 1024, [])) + const { iterator } = await generateVmdkData('result.vmdk', 1024 * 1024 * 1024, 1024 * 1024, []) + const readStream = asyncIteratorToStream(iterator) const pipe = readStream.pipe(createWriteStream('result.vmdk')) await fromEvent(pipe, 'finish') await execa('qemu-img', ['check', 'result.vmdk']) From 9957c3ad471449ccae38d90f6cf345d9de15fa78 Mon Sep 17 00:00:00 2001 From: florent Beauchamp Date: Wed, 25 Jan 2023 15:51:18 +0100 Subject: [PATCH 2/5] changelog --- CHANGELOG.unreleased.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 292b469c08e..9092fd9d39f 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -11,6 +11,8 @@ > Users must be able to say: “I had this issue, happy to know it's fixed” +- [Ova export] Better computation of overprovisionning for very sparse disks (PR [#6639](https://github.com/vatesfr/xen-orchestra/pull/6639)) + ### Packages to release > When modifying a package, add it here with its release type. @@ -27,6 +29,7 @@ +- xo-vmdk-to-vhd patch - xo-server-upload-ova patch From 33f0048594461cbbcd9c9cba3b4f3a27219fbf11 Mon Sep 17 00:00:00 2001 From: florent Beauchamp Date: Wed, 25 Jan 2023 15:59:09 +0100 Subject: [PATCH 3/5] fix tests --- packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js b/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js index 19a7e375501..ba935421963 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js @@ -103,9 +103,8 @@ test('Can generate a small VMDK file', async () => { ] const fileName = 'result.vmdk' const geometry = { sectorsPerTrackCylinder: 63, heads: 16, cylinders: 10402 } - const readStream = asyncIteratorToStream( - await generateVmdkData(fileName, 2 * blockSize, blockSize, blockGenerator, geometry) - ) + const { iterator } = await await generateVmdkData(fileName, 2 * blockSize, blockSize, blockGenerator, geometry) + const readStream = asyncIteratorToStream(iterator) const pipe = readStream.pipe(createWriteStream(fileName)) await fromEvent(pipe, 'finish') From bc7116f12cf49219439c13421b3e5d95f386618f Mon Sep 17 00:00:00 2001 From: Florent Beauchamp Date: Mon, 30 Jan 2023 14:05:22 +0100 Subject: [PATCH 4/5] cleanup --- packages/xo-vmdk-to-vhd/src/vmdk-generate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-generate.js b/packages/xo-vmdk-to-vhd/src/vmdk-generate.js index 694446efa68..15ae43267f4 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-generate.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-generate.js @@ -165,8 +165,8 @@ ddb.geometry.cylinders = "${geometry.cylinders}" roundToSector(headerData.grainDirectoryEntries * 4) + SECTOR_SIZE + // MARKER_FOOTER SECTOR_SIZE + // stream optimizedheader - SECTOR_SIZE - ) // MARKER_EOS + SECTOR_SIZE // MARKER_EOS + ) } function* padding() { From beb15d12be19a63dd571a038249f6642b68eae87 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Mon, 30 Jan 2023 14:15:16 +0100 Subject: [PATCH 5/5] Update CHANGELOG.unreleased.md --- CHANGELOG.unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 9092fd9d39f..648ac1fe3e3 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -11,7 +11,7 @@ > Users must be able to say: “I had this issue, happy to know it's fixed” -- [Ova export] Better computation of overprovisionning for very sparse disks (PR [#6639](https://github.com/vatesfr/xen-orchestra/pull/6639)) +- [Ova export] Better computation of overprovisioning for very sparse disks (PR [#6639](https://github.com/vatesfr/xen-orchestra/pull/6639)) ### Packages to release