diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c74a5d0c..abc4c8ec 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,7 +46,6 @@ jobs: run: npm test functional-test-cli: - if: false runs-on: ubuntu-22.04 defaults: diff --git a/cli/src/commands/list.ts b/cli/src/commands/list.ts index bc4bf222..3fa9f2f7 100644 --- a/cli/src/commands/list.ts +++ b/cli/src/commands/list.ts @@ -28,5 +28,6 @@ export const listFilesCommand = new Command('list') console.log('Targeting website at address', sc.address) const files = await listFiles(provider, sc) + console.log(`Total of ${files.length} files:`) files.sort().forEach((f) => console.log(f)) }) diff --git a/cli/src/commands/upload.ts b/cli/src/commands/upload.ts index 97c2bbbd..cd944be3 100644 --- a/cli/src/commands/upload.ts +++ b/cli/src/commands/upload.ts @@ -37,10 +37,14 @@ export const uploadCommand = new Command('upload') batches: [], chunks: [], fileInits: [], + filesToDelete: [], + metadatas: [], + metadatasToDelete: [], chunkSize: chunkSize, websiteDirPath: websiteDirPath, skipConfirm: options.yes, currentTotalEstimation: 0n, + maxConcurrentOps: 4, minimalFees: await provider.client.getMinimalFee(), } diff --git a/cli/src/lib/website/filesInit.ts b/cli/src/lib/website/filesInit.ts index 1663f80d..0f5360aa 100644 --- a/cli/src/lib/website/filesInit.ts +++ b/cli/src/lib/website/filesInit.ts @@ -2,9 +2,11 @@ import { Args, Mas, MAX_GAS_CALL, + MIN_GAS_CALL, minBigInt, Operation, SmartContract, + strToBytes, U32, } from '@massalabs/massa-web3' import { storageCostForEntry } from '../utils/storage' @@ -16,10 +18,70 @@ import { globalMetadataKey, } from './storageKeys' import { FileDelete } from './models/FileDelete' +import { maxBigInt } from '../../tasks/utils' const functionName = 'filesInit' const batchSize = 20 +/** + * Divide the files, filesToDelete, metadatas, and metadatasToDelete into multiple batches + * @param files - Array of FileInit instances + * @param filesToDelete - Array of FileDelete instances + * @param metadatas - Array of Metadata instances + * @param metadatasToDelete - Array of Metadata instances to delete + * @param batchSize - Maximum number of elements in each batch + * @returns - Array of Batch instances + */ +function createBatches( + files: FileInit[], + filesToDelete: FileDelete[], + metadatas: Metadata[], + metadatasToDelete: Metadata[] +): Batch[] { + const batches: Batch[] = [] + + let currentBatch = new Batch([], [], [], []) + + const addBatch = () => { + if (Object.values(currentBatch).some((v) => v.length > 0)) { + batches.push(currentBatch) + currentBatch = new Batch([], [], [], []) + } + } + + for (const file of files) { + if (currentBatch.fileInits.length >= batchSize) { + addBatch() + } + currentBatch.fileInits.push(file) + } + + for (const fileDelete of filesToDelete) { + if (currentBatch.fileDeletes.length >= batchSize) { + addBatch() + } + currentBatch.fileDeletes.push(fileDelete) + } + + for (const metadata of metadatas) { + if (currentBatch.metadatas.length >= batchSize) { + addBatch() + } + currentBatch.metadatas.push(metadata) + } + + for (const metadataDelete of metadatasToDelete) { + if (currentBatch.metadataDeletes.length >= batchSize) { + addBatch() + } + currentBatch.metadataDeletes.push(metadataDelete) + } + + addBatch() + + return batches +} + /** * Send the filesInits to the smart contract * @param sc - SmartContract instance @@ -33,44 +95,27 @@ export async function sendFilesInits( files: FileInit[], filesToDelete: FileDelete[], metadatas: Metadata[], - metadatasToDelete: Metadata[] + metadatasToDelete: Metadata[], + minimalFees: bigint = Mas.fromString('0.01') ): Promise { - const fileInitsBatches: FileInit[][] = [] - const operations: Operation[] = [] + const batches: Batch[] = createBatches( + files, + filesToDelete, + metadatas, + metadatasToDelete + ) - for (let i = 0; i < files.length; i += batchSize) { - fileInitsBatches.push(files.slice(i, i + batchSize)) - } + const operations: Operation[] = [] - for (const batch of fileInitsBatches) { - const coins = await filesInitCost( - sc, - batch, - filesToDelete, - metadatas, - metadatasToDelete - ) - const gas = await estimatePrepareGas( - sc, - batch, - filesToDelete, - metadatas, - metadatasToDelete - ) - const args = new Args() - .addSerializableObjectArray(batch) - .addSerializableObjectArray(filesToDelete) - .addSerializableObjectArray(metadatas) - .addSerializableObjectArray(metadatasToDelete) - .serialize() + for (const batch of batches) { + const coins = await batch.batchCost(sc) + const gas = await batch.estimateGas(sc) + const args = batch.serialize() const op = await sc.call(functionName, args, { - coins: coins, + coins: coins <= 0n ? 0n : coins, maxGas: gas, - fee: - BigInt(gas) > BigInt(Mas.fromString('0.01')) - ? BigInt(gas) - : BigInt(Mas.fromString('0.01')), + fee: gas > minimalFees ? gas : minimalFees, }) operations.push(op) @@ -122,7 +167,7 @@ export async function filesInitCost( return ( acc + storageCostForEntry( - BigInt(globalMetadataKey(metadata.key).length), + BigInt(globalMetadataKey(strToBytes(metadata.key)).length), BigInt(metadata.value.length + 4) ) ) @@ -132,7 +177,7 @@ export async function filesInitCost( return ( acc + storageCostForEntry( - BigInt(globalMetadataKey(metadata.key).length), + BigInt(globalMetadataKey(strToBytes(metadata.key)).length), BigInt(metadata.value.length + 4) ) ) @@ -148,13 +193,17 @@ export async function filesInitCost( } /** - * Estimate the gas cost for the operation + * Estimate the gas cost for the prepare operation * Required until https://github.com/massalabs/massa/issues/4742 is fixed * @param sc - SmartContract instance - * @param files - Array of PreStore instances + * @param files - Array of FileInit instances + * @param filesToDelete - Array of FileDelete instances + * @param metadatas - Array of Metadata instances + * @param metadatasToDelete - Array of Metadata instances to delete + * * @returns - Estimated gas cost for the operation */ -export async function estimatePrepareGas( +async function estimatePrepareGas( sc: SmartContract, files: FileInit[], filesToDelete: FileDelete[], @@ -176,14 +225,65 @@ export async function estimatePrepareGas( .serialize() const result = await sc.read(functionName, args, { - coins: coins, + coins: coins <= 0n ? 0n : coins, maxGas: MAX_GAS_CALL, }) if (result.info.error) { + console.error(result.info) throw new Error(result.info.error) } const gasCost = BigInt(result.info.gasCost) + const numberOfElements = BigInt( + files.length + + filesToDelete.length + + metadatas.length + + metadatasToDelete.length + ) + + return minBigInt( + maxBigInt(gasCost * numberOfElements, MIN_GAS_CALL), + MAX_GAS_CALL + ) +} + +/** + * Represents parameters for the filesInit function + */ +class Batch { + constructor( + public fileInits: FileInit[], + public fileDeletes: FileDelete[], + public metadatas: Metadata[], + public metadataDeletes: Metadata[] + ) {} + + serialize(): Uint8Array { + return new Args() + .addSerializableObjectArray(this.fileInits) + .addSerializableObjectArray(this.fileDeletes) + .addSerializableObjectArray(this.metadatas) + .addSerializableObjectArray(this.metadataDeletes) + .serialize() + } - return minBigInt(gasCost * BigInt(files.length), MAX_GAS_CALL) + batchCost(sc: SmartContract): Promise { + return filesInitCost( + sc, + this.fileInits, + this.fileDeletes, + this.metadatas, + this.metadataDeletes + ) + } + + estimateGas(sc: SmartContract): Promise { + return estimatePrepareGas( + sc, + this.fileInits, + this.fileDeletes, + this.metadatas, + this.metadataDeletes + ) + } } diff --git a/cli/src/lib/website/models/FileDelete.ts b/cli/src/lib/website/models/FileDelete.ts index a23a9775..4975cd80 100644 --- a/cli/src/lib/website/models/FileDelete.ts +++ b/cli/src/lib/website/models/FileDelete.ts @@ -1,7 +1,13 @@ import { Args, DeserializedResult, Serializable } from '@massalabs/massa-web3' +import { sha256 } from 'js-sha256' export class FileDelete implements Serializable { - constructor(public hashLocation: Uint8Array = new Uint8Array(0)) {} + constructor( + public location: string, + public hashLocation: Uint8Array = new Uint8Array( + sha256.arrayBuffer(location) + ) + ) {} serialize(): Uint8Array { return new Args().addUint8Array(this.hashLocation).serialize() diff --git a/cli/src/lib/website/models/Metadata.ts b/cli/src/lib/website/models/Metadata.ts index 6e660067..e1e7b3a9 100644 --- a/cli/src/lib/website/models/Metadata.ts +++ b/cli/src/lib/website/models/Metadata.ts @@ -2,22 +2,19 @@ import { Args, DeserializedResult, Serializable } from '@massalabs/massa-web3' export class Metadata implements Serializable { constructor( - public key: Uint8Array = new Uint8Array(0), - public value: Uint8Array = new Uint8Array(0) + public key: string = '', + public value: string = '' ) {} serialize(): Uint8Array { - return new Args() - .addUint8Array(this.key) - .addUint8Array(this.value) - .serialize() + return new Args().addString(this.key).addString(this.value).serialize() } deserialize(data: Uint8Array, offset: number): DeserializedResult { const args = new Args(data, offset) - this.key = args.nextUint8Array() - this.value = args.nextUint8Array() + this.key = args.nextString() + this.value = args.nextString() return { instance: this, offset: args.getOffset() } } diff --git a/cli/src/tasks/estimations.ts b/cli/src/tasks/estimations.ts index 8d7cdd50..89784155 100644 --- a/cli/src/tasks/estimations.ts +++ b/cli/src/tasks/estimations.ts @@ -50,6 +50,7 @@ export function showEstimatedCost(): ListrTask { } ctx.currentTotalEstimation += opFees + ctx.currentTotalEstimation += totalEstimatedGas }, rendererOptions: { outputBar: Infinity, diff --git a/cli/src/tasks/prepareChunk.ts b/cli/src/tasks/prepareChunk.ts index 01063d55..8bf41255 100644 --- a/cli/src/tasks/prepareChunk.ts +++ b/cli/src/tasks/prepareChunk.ts @@ -8,12 +8,13 @@ import { Provider, SmartContract, U32 } from '@massalabs/massa-web3' import { batcher } from '../lib/batcher' import { divideIntoChunks, toChunkPosts } from '../lib/website/chunk' -import { getFileFromAddress } from '../lib/website/read' +import { getFileFromAddress, listFiles } from '../lib/website/read' import { FILE_TAG, fileChunkCountKey } from '../lib/website/storageKeys' import { FileChunkPost } from '../lib/website/models/FileChunkPost' import { FileInit } from '../lib/website/models/FileInit' +import { FileDelete } from '../lib/website/models/FileDelete' import { UploadCtx } from './tasks' /** @@ -24,12 +25,20 @@ export function prepareBatchesTask(): ListrTask { return { title: 'Preparing batches', task: async (ctx: UploadCtx, task) => { - const { chunks, fileInits } = await prepareChunks( + const { chunks, fileInits, localFiles } = await prepareChunks( ctx.provider, ctx.websiteDirPath, ctx.chunkSize, ctx.sc ) + + if (ctx.sc) { + const filesInSC = await listFiles(ctx.provider, ctx.sc) + ctx.filesToDelete = filesInSC + .filter((file) => !localFiles.includes(file)) + .map((file) => new FileDelete(file)) + } + ctx.batches = batcher(chunks, ctx.chunkSize) ctx.fileInits = ctx.sc ? await filterUselessFileInits(ctx.provider, ctx.sc.address, fileInits) @@ -44,7 +53,11 @@ export function prepareBatchesTask(): ListrTask { } } - task.output = `Total of ${fileInits.length} files, only ${ctx.fileInits.length} require update` + task.output = `Total of ${fileInits.length} files, ${ctx.fileInits.length} require update` + task.output = `${ctx.filesToDelete.length} files will be deleted:` + for (const file of ctx.filesToDelete) { + task.output = `- ${file.location}` + } task.output = `Total of ${chunks.length} chunks divided into ${ctx.batches.length} batches` }, rendererOptions: { @@ -66,12 +79,18 @@ async function prepareChunks( chunkSize: number, sc?: SmartContract, basePath: string = path -): Promise<{ chunks: FileChunkPost[]; fileInits: FileInit[] }> { +): Promise<{ + chunks: FileChunkPost[] + fileInits: FileInit[] + localFiles: string[] +}> { const files = readdirSync(path) const chunks: FileChunkPost[] = [] const fileInits: FileInit[] = [] + const localFiles: string[] = [] + for (const file of files) { const fullPath = join(path, file) const stats = statSync(fullPath) @@ -87,10 +106,13 @@ async function prepareChunks( ) chunks.push(...result.chunks) fileInits.push(...result.fileInits) + localFiles.push(...result.localFiles) } else if (stats.isFile()) { const data = readFileSync(fullPath) const relativePath = relative(basePath, fullPath) + localFiles.push(relativePath) + if (!(await requiresUpdate(provider, relativePath, data, sc))) { continue } @@ -104,7 +126,7 @@ async function prepareChunks( } } - return { chunks, fileInits } + return { chunks, fileInits, localFiles } } /** diff --git a/cli/src/tasks/prepareUpload.ts b/cli/src/tasks/prepareUpload.ts index d28deaec..b61d65ea 100644 --- a/cli/src/tasks/prepareUpload.ts +++ b/cli/src/tasks/prepareUpload.ts @@ -1,8 +1,9 @@ import { ListrEnquirerPromptAdapter } from '@listr2/prompt-adapter-enquirer' -import { formatMas, OperationStatus } from '@massalabs/massa-web3' +import { formatMas, OperationStatus, U64 } from '@massalabs/massa-web3' import { ListrTask } from 'listr2' import { filesInitCost, sendFilesInits } from '../lib/website/filesInit' +import { Metadata } from '../lib/website/models/Metadata' import { UploadCtx } from './tasks' @@ -14,19 +15,28 @@ export function prepareUploadTask(): ListrTask { return { title: 'Prepare upload', task: (ctx: UploadCtx, task) => { - if (ctx.fileInits.length === 0) { + if (ctx.fileInits.length === 0 && ctx.filesToDelete.length === 0) { task.skip('All files are ready for upload') return } + const utcNowDate = U64.fromNumber(Math.floor(Date.now() / 1000)) + + ctx.metadatas.push(new Metadata('LAST_UPDATE', utcNowDate.toString())) + return task.newListr( [ { title: 'Confirm SC preparation', task: async (ctx, subTask) => { const cost = - (await filesInitCost(ctx.sc, ctx.fileInits, [], [], [])) + - ctx.minimalFees + (await filesInitCost( + ctx.sc, + ctx.fileInits, + ctx.filesToDelete, + ctx.metadatas, + ctx.metadatasToDelete + )) + ctx.minimalFees subTask.output = `SC preparation costs ${formatMas(cost)} MAS (including ${formatMas(ctx.minimalFees)} MAS of minimal fees)` if (!ctx.skipConfirm) { @@ -57,9 +67,10 @@ export function prepareUploadTask(): ListrTask { const operations = await sendFilesInits( ctx.sc, ctx.fileInits, - [], - [], - [] + ctx.filesToDelete, + ctx.metadatas, + ctx.metadatasToDelete, + ctx.minimalFees ) const results = await Promise.all( diff --git a/cli/src/tasks/tasks.ts b/cli/src/tasks/tasks.ts index b7e1328a..10b224dd 100644 --- a/cli/src/tasks/tasks.ts +++ b/cli/src/tasks/tasks.ts @@ -2,7 +2,9 @@ import { Provider, SmartContract } from '@massalabs/massa-web3' import { Batch } from '../lib/batcher' import { FileChunkPost } from '../lib/website/models/FileChunkPost' +import { FileDelete } from '../lib/website/models/FileDelete' import { FileInit } from '../lib/website/models/FileInit' +import { Metadata } from '../lib/website/models/Metadata' export interface UploadCtx { provider: Provider @@ -14,9 +16,14 @@ export interface UploadCtx { chunks: FileChunkPost[] fileInits: FileInit[] + filesToDelete: FileDelete[] + metadatas: Metadata[] + metadatasToDelete: Metadata[] batches: Batch[] chunkSize: number minimalFees: bigint + + maxConcurrentOps: number } export interface DeleteCtx { diff --git a/cli/src/tasks/upload.ts b/cli/src/tasks/upload.ts index e2ecdc66..eb1bbe81 100644 --- a/cli/src/tasks/upload.ts +++ b/cli/src/tasks/upload.ts @@ -33,14 +33,12 @@ export function confirmUploadTask(): ListrTask { * @returns a Listr task to upload batches */ export function uploadBatchesTask(): ListrTask { - const maxConcurrentOps = 4 - return { title: 'Uploading batches', skip: (ctx: UploadCtx) => ctx.batches.length === 0, task: async (ctx, task) => { const uploadManager = new UploadManager(ctx.uploadBatches, { - maxConcurrentOps: maxConcurrentOps, + maxConcurrentOps: ctx.maxConcurrentOps, }) await uploadManager.startUpload(ctx.sc, () => { diff --git a/cli/src/tasks/utils.ts b/cli/src/tasks/utils.ts index 12d88c7c..a8a5a491 100644 --- a/cli/src/tasks/utils.ts +++ b/cli/src/tasks/utils.ts @@ -10,3 +10,13 @@ export function formatBytes(bytes: number): string { const i = Math.floor(Math.log(bytes) / Math.log(k)) return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i] } + +/** + * Returns the maximum of two bigints + * @param a - the first bigint + * @param b - the second bigint + * @returns the maximum of the two bigints + */ +export function maxBigInt(a: bigint, b: bigint): bigint { + return a > b ? a : b +} diff --git a/server/int/api/middlewares.go b/server/int/api/middlewares.go index 5bcb9bbe..957165b6 100644 --- a/server/int/api/middlewares.go +++ b/server/int/api/middlewares.go @@ -120,46 +120,59 @@ func resolveAddress(subdomain string, network msConfig.NetworkInfos) (string, er return domainTarget, nil } -func getWebsiteResource(network *msConfig.NetworkInfos, websiteAddress, resourceName string) ([]byte, string, error) { - logger.Debugf("Getting website %s resource %s", websiteAddress, resourceName) - - content, notFound, err := webmanager.GetWebsiteResource(network, websiteAddress, resourceName) +func resolveResourceName(network *msConfig.NetworkInfos, websiteAddress, resourceName string) (string, error) { + exists, err := webmanager.ResourceExistsOnChain(network, websiteAddress, resourceName) if err != nil { - if !notFound { - return nil, "", fmt.Errorf("failed to get website: %w", err) - } + return "", fmt.Errorf("failed to check if resource exists: %w", err) + } + if !exists { + logger.Warnf("Resource %s not found in website %s", resourceName, websiteAddress) // Handling missing .html extension - if notFound && !strings.HasSuffix(resourceName, ".html") { - logger.Warnf("Failed to get file %s from website: %v", resourceName, err) + if !strings.HasSuffix(resourceName, ".html") { resourceName += ".html" - content, notFound, err = webmanager.GetWebsiteResource(network, websiteAddress, resourceName) + exists, err = webmanager.ResourceExistsOnChain(network, websiteAddress, resourceName) if err != nil { - if notFound { - return nil, "", fmt.Errorf("could not find %s in website: %w", resourceName, err) - } + return "", fmt.Errorf("failed to check if resource exists: %w", err) + } - return nil, "", fmt.Errorf("failed to get file from website: %w", err) + if exists { + return resourceName, nil } } // Handling Single Page Apps - if notFound && resourceName != "index.html" { - logger.Warnf("Failed to get file %s from zip: %v", resourceName, err) + if resourceName != "index.html" { resourceName = "index.html" - content, notFound, err = webmanager.GetWebsiteResource(network, websiteAddress, resourceName) + exists, err = webmanager.ResourceExistsOnChain(network, websiteAddress, resourceName) if err != nil { - if notFound { - return nil, "", fmt.Errorf("could not find index.html in website: %w", err) - } + return "", fmt.Errorf("failed to check if resource exists: %w", err) + } - return nil, "", fmt.Errorf("failed to get file from zip: %w", err) + if exists { + return resourceName, nil } } } + return resourceName, nil +} + +func getWebsiteResource(network *msConfig.NetworkInfos, websiteAddress, resourceName string) ([]byte, string, error) { + logger.Debugf("Getting website %s resource %s", websiteAddress, resourceName) + + resourceName, err := resolveResourceName(network, websiteAddress, resourceName) + if err != nil { + return nil, "", fmt.Errorf("failed to resolve resource name: %w", err) + } + + content, err := webmanager.GetWebsiteResource(network, websiteAddress, resourceName) + if err != nil { + return nil, "", fmt.Errorf("failed to get website %s resource %s: %w", websiteAddress, resourceName, err) + } + contentType := ContentType(resourceName, content) logger.Debugf("Got website %s resource %s with content type %s", websiteAddress, resourceName, contentType) diff --git a/server/pkg/webmanager/manager.go b/server/pkg/webmanager/manager.go index 9557d8ec..f2e762a8 100644 --- a/server/pkg/webmanager/manager.go +++ b/server/pkg/webmanager/manager.go @@ -2,8 +2,6 @@ package webmanager import ( "fmt" - "strings" - "time" "github.com/massalabs/deweb-server/pkg/cache" "github.com/massalabs/deweb-server/pkg/website" @@ -14,55 +12,48 @@ import ( const cacheDir = "./websitesCache/" // getWebsiteResource fetches a resource from a website and returns its content. -func GetWebsiteResource(network *msConfig.NetworkInfos, websiteAddress, resourceName string) ([]byte, bool, error) { +func GetWebsiteResource(network *msConfig.NetworkInfos, websiteAddress, resourceName string) ([]byte, error) { logger.Debugf("Getting website %s resource %s", websiteAddress, resourceName) content, err := RequestFile(websiteAddress, network, resourceName) if err != nil { - if strings.Contains(err.Error(), "not found") { - return nil, true, fmt.Errorf("failed to get file %s from website %s: %w", resourceName, websiteAddress, err) - } - - return nil, false, fmt.Errorf("failed to get file %s from website %s: %w", resourceName, websiteAddress, err) + return nil, fmt.Errorf("failed to get file %s from website %s: %w", resourceName, websiteAddress, err) } logger.Debugf("Resource %s from %s successfully retrieved", resourceName, websiteAddress) - return content, false, nil + return content, nil } // RequestFile fetches a website and caches it, or retrieves it from the cache if already present. func RequestFile(scAddress string, networkInfo *msConfig.NetworkInfos, resourceName string) ([]byte, error) { cache, err := cache.NewCache(cacheDir) if err != nil { - return nil, fmt.Errorf("failed to create cache: %w", err) + logger.Errorf("Failed to create cache: %v", err) } - // TODO: LastUpdateTimestamp is not yet implemented in the new SC - // lastUpdatedUint, err := website.GetLastUpdateTimestamp(networkInfo, scAddress) - // if err != nil { - // return nil, fmt.Errorf("failed to get last update timestamp: %w", err) - // } - - // lastUpdated := time.UnixMilli(int64(lastUpdatedUint)) - - if cache.IsPresent(scAddress, resourceName) { + if cache != nil && cache.IsPresent(scAddress, resourceName) { logger.Debugf("Resource %s from %s present in cache", resourceName, scAddress) - // isOutdated, err := isFileOutdated(resourceName, lastUpdated) - // if err != nil { - // return nil, fmt.Errorf("failed to check if file is outdated: %w", err) - // } - - // if !isOutdated { - content, err := cache.Read(scAddress, resourceName) + isOutdated, err := isFileOutdated(cache, networkInfo, scAddress, resourceName) if err != nil { - return nil, fmt.Errorf("failed to read cached resource %s from %s: %w", resourceName, scAddress, err) + logger.Warnf("Failed to check if file is outdated: %v", err) + } else { + if !isOutdated { + content, err := cache.Read(scAddress, resourceName) + if err != nil { + logger.Warnf("Failed to read cached resource %s from %s: %v", resourceName, scAddress, err) + } else { + return content, nil + } + } else { + if err = cache.Delete(scAddress, resourceName); err != nil { + logger.Warnf("Failed to delete outdated resource %s from %s: %v", resourceName, scAddress, err) + } + } } - return content, nil - // } - // logger.Warnf("website %s is outdated, fetching...", resourceName) + logger.Warnf("website %s is outdated, fetching...", resourceName) } logger.Debugf("Website %s not found in cache or not up to date, fetching...", scAddress) @@ -79,32 +70,59 @@ func RequestFile(scAddress string, networkInfo *msConfig.NetworkInfos, resourceN func fetchAndCache(networkInfo *msConfig.NetworkInfos, scAddress string, cache *cache.Cache, resourceName string) ([]byte, error) { websiteBytes, err := website.Fetch(networkInfo, scAddress, resourceName) if err != nil { - if website.IsNotFoundError(err, resourceName) { - return nil, fmt.Errorf("website %s not found: %w", scAddress, err) - } - return nil, fmt.Errorf("failed to fetch %s from %s: %w", resourceName, scAddress, err) } logger.Debugf("%s: %s successfully fetched with size: %d bytes", scAddress, resourceName, len(websiteBytes)) - if err := cache.Save(scAddress, resourceName, websiteBytes); err != nil { - return nil, fmt.Errorf("failed to save %s to %s cache: %w", resourceName, scAddress, err) - } + if cache != nil { + err := cache.Save(scAddress, resourceName, websiteBytes) + if err != nil { + return nil, fmt.Errorf("failed to save %s to %s cache: %w", resourceName, scAddress, err) + } - logger.Infof("%s: %s successfully written to cache", scAddress, resourceName) + logger.Infof("%s: %s successfully written to cache", scAddress, resourceName) + } return websiteBytes, nil } // Compares the last updated timestamp to the file's last modified timestamp. +// Returns true if the file is outdated or if an error occurred. Otherwise, returns false. // //nolint:unused -func isFileOutdated(cache *cache.Cache, scAddress string, fileName string, lastUpdated time.Time) (bool, error) { +func isFileOutdated(cache *cache.Cache, networkInfo *msConfig.NetworkInfos, scAddress string, fileName string) (bool, error) { + lastUpdated, err := website.GetLastUpdateTimestamp(networkInfo, scAddress) + if err != nil { + return true, fmt.Errorf("failed to get last update timestamp: %w", err) + } + lastModified, err := cache.GetLastModified(scAddress, fileName) if err != nil { - return false, fmt.Errorf("failed to get file info: %w", err) + return true, fmt.Errorf("failed to get file info: %w", err) + } + + return lastModified.Before(*lastUpdated), nil +} + +func ResourceExistsOnChain(network *msConfig.NetworkInfos, websiteAddress, filePath string) (bool, error) { + logger.Debugf("Checking if file %s exists on chain for website %s", filePath, websiteAddress) + + isPresent, err := website.FilePathExists(network, websiteAddress, filePath) + if err != nil { + return false, fmt.Errorf("checking if file is present on chain: %w", err) + } + + return isPresent, nil +} + +func ResourceExistsInCache(scAddress string, resourceName string) (bool, error) { + logger.Debugf("Checking if resource %s exists in cache for website %s", resourceName, scAddress) + + cache, err := cache.NewCache(cacheDir) + if err != nil { + return false, fmt.Errorf("failed to create cache: %w", err) } - return lastModified.Before(lastUpdated), nil + return cache.IsPresent(scAddress, resourceName), nil } diff --git a/server/pkg/website/const.go b/server/pkg/website/const.go index 59368d05..3ff24fe0 100644 --- a/server/pkg/website/const.go +++ b/server/pkg/website/const.go @@ -1,12 +1,7 @@ package website const ( - fileTag = "\000" - chunkTag = "\001" - chunkNumberTag = "\002" - matadataTag = "\003" - filesPathList = "\005" - ownerKey = "OWNER" + ownerKey = "OWNER" ChunkSize = 64_000 // 64KB ) diff --git a/server/pkg/website/read.go b/server/pkg/website/read.go index 686c63a9..c393f2b7 100644 --- a/server/pkg/website/read.go +++ b/server/pkg/website/read.go @@ -1,8 +1,11 @@ package website import ( + "bytes" "crypto/sha256" "fmt" + "strconv" + "time" "github.com/massalabs/deweb-server/pkg/website/storagekeys" "github.com/massalabs/station/int/config" @@ -12,14 +15,26 @@ import ( ) const ( - datastoreBatchSize = 64 - notFoundErrorTemplate = "no chunks found for file %s" + datastoreBatchSize = 64 + notFoundErrorTemplate = "no chunks found for file %s" + lastUpdateTimestampKey = "LAST_UPDATE" ) // Fetch retrieves the complete data of a website as bytes. func Fetch(network *config.NetworkInfos, websiteAddress string, filePath string) ([]byte, error) { client := node.NewClient(network.NodeURL) + isPresent, err := FilePathExists(network, websiteAddress, filePath) + if err != nil { + return nil, fmt.Errorf("checking if file is present on chain: %w", err) + } + + if isPresent { + logger.Debugf("File '%s' is present on chain", filePath) + } else { + return nil, fmt.Errorf("file '%s' not found on chain", filePath) + } + chunkNumber, err := GetNumberOfChunks(client, websiteAddress, filePath) if err != nil { return nil, fmt.Errorf("fetching number of chunks: %w", err) @@ -28,7 +43,7 @@ func Fetch(network *config.NetworkInfos, websiteAddress string, filePath string) logger.Debugf("Number of chunks for file '%s': %d", filePath, chunkNumber) if chunkNumber == 0 { - return nil, fmt.Errorf(notFoundErrorTemplate, filePath) + return nil, fmt.Errorf("no chunks found for file '%s'", filePath) } dataStore, err := fetchAllChunks(client, websiteAddress, filePath, chunkNumber) @@ -39,11 +54,6 @@ func Fetch(network *config.NetworkInfos, websiteAddress string, filePath string) return dataStore, nil } -// IsNotFoundError checks if the error is a not found error. -func IsNotFoundError(err error, fileName string) bool { - return fmt.Sprintf(notFoundErrorTemplate, fileName) == err.Error() -} - // GetNumberOfChunks fetches and returns the number of chunks for the website. func GetNumberOfChunks(client *node.Client, websiteAddress string, filePath string) (int32, error) { filePathHash := sha256.Sum256([]byte(filePath)) @@ -72,20 +82,46 @@ func GetFilesPathList( client *node.Client, websiteAddress string, ) ([]string, error) { - filesPathListResponse, err := node.FetchDatastoreEntry(client, websiteAddress, []byte(filesPathList)) + filteredKeys, err := getFileLocationKeys(client, websiteAddress) if err != nil { - return nil, fmt.Errorf("fetching website files path list: %w", err) + return nil, fmt.Errorf("fetching website file location keys: %w", err) } - if filesPathListResponse.FinalValue == nil { - return nil, nil + filesPathListResponse, err := node.ContractDatastoreEntries(client, websiteAddress, filteredKeys) + if err != nil { + return nil, fmt.Errorf("fetching website files path list: %w", err) } - filesPathList := convert.ToStringArray(filesPathListResponse.FinalValue) + filesPathList := make([]string, len(filesPathListResponse)) + + for i, entry := range filesPathListResponse { + filesPathList[i] = string(entry.FinalValue) + } return filesPathList, nil } +// getFileLocationKeys fetches and returns the keys for the file locations. +func getFileLocationKeys(client *node.Client, websiteAddress string) ([][]byte, error) { + addressesInfo, err := node.Addresses(client, []string{websiteAddress}) + if err != nil { + return nil, fmt.Errorf("converting website address: %w", err) + } + + addressInfo := addressesInfo[0] + keys := addressInfo.FinalDatastoreKeys + + var filteredKeys [][]byte + + for _, key := range keys { + if len(key) > len(storagekeys.FileLocationTag()) && bytes.Equal(key[:len(storagekeys.FileLocationTag())], storagekeys.FileLocationTag()) { + filteredKeys = append(filteredKeys, key) + } + } + + return filteredKeys, nil +} + // fetchAllChunks retrieves all chunks of data for the website. func fetchAllChunks(client *node.Client, websiteAddress string, filePath string, chunkNumber int32) ([]byte, error) { filePathHash := sha256.Sum256([]byte(filePath)) @@ -115,6 +151,10 @@ func fetchAllChunks(client *node.Client, websiteAddress string, filePath string, } for _, entry := range response { + if len(entry.FinalValue) == 0 { + return nil, fmt.Errorf("empty chunk") + } + dataStore = append(dataStore, entry.FinalValue...) } } @@ -134,44 +174,48 @@ func GetOwner(network *config.NetworkInfos, websiteAddress string) (string, erro return string(ownerResponse.FinalValue), nil } -// TODO: Update those functions once implemented in the new SC +// GetLastUpdateTimestamp retrieves the last update timestamp of the website. +func GetLastUpdateTimestamp(network *config.NetworkInfos, websiteAddress string) (*time.Time, error) { + client := node.NewClient(network.NodeURL) -// func GetFirstCreationTimestamp(network *config.NetworkInfos, websiteAddress string) (uint64, error) { -// client := node.NewClient(network.NodeURL) + lastUpdateTimestampResponse, err := node.FetchDatastoreEntry(client, websiteAddress, storagekeys.GlobalMetadataKey(convert.ToBytes(lastUpdateTimestampKey))) + if err != nil { + return nil, fmt.Errorf("fetching website last update timestamp: %w", err) + } -// firstCreationTimestampResponse, err := node.FetchDatastoreEntry(client, websiteAddress, convert.ToBytes(firstCreationTimestampKey)) -// if err != nil { -// return 0, fmt.Errorf("fetching website first creation timestamp: %w", err) -// } + if lastUpdateTimestampResponse.FinalValue == nil { + return nil, nil + } -// if firstCreationTimestampResponse.FinalValue == nil { -// return 0, nil -// } + timestampStr := string(lastUpdateTimestampResponse.FinalValue) -// castedFCTimestamp, err := convert.BytesToU64(firstCreationTimestampResponse.FinalValue) -// if err != nil { -// return 0, fmt.Errorf("converting website first creation timestamp: %w", err) -// } + castedLUTimestamp, err := strconv.ParseUint(timestampStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("converting website last update timestamp: %w", err) + } -// return castedFCTimestamp, nil -// } + timestamp := time.Unix(int64(castedLUTimestamp), 0) -// func GetLastUpdateTimestamp(network *config.NetworkInfos, websiteAddress string) (uint64, error) { -// client := node.NewClient(network.NodeURL) + return ×tamp, nil +} -// lastUpdateTimestampResponse, err := node.FetchDatastoreEntry(client, websiteAddress, convert.ToBytes(lastUpdateTimestampKey)) -// if err != nil { -// return 0, fmt.Errorf("fetching website last update timestamp: %w", err) -// } +// Check if the requested filePath exists in the SC FilesPathList +func FilePathExists(network *config.NetworkInfos, websiteAddress string, filePath string) (bool, error) { + client := node.NewClient(network.NodeURL) + if client == nil { + return false, fmt.Errorf("failed to create node client") + } -// if lastUpdateTimestampResponse.FinalValue == nil { -// return 0, nil -// } + files, err := GetFilesPathList(client, websiteAddress) + if err != nil { + return false, fmt.Errorf("failed to get files path list: %w", err) + } -// castedLUTimestamp, err := convert.BytesToU64(lastUpdateTimestampResponse.FinalValue) -// if err != nil { -// return 0, fmt.Errorf("converting website last update timestamp: %w", err) -// } + for _, file := range files { + if file == filePath { + return true, nil + } + } -// return castedLUTimestamp, nil -// } + return false, nil +} diff --git a/server/pkg/website/storagekeys/tags.go b/server/pkg/website/storagekeys/tags.go index 548f0260..7e686e0b 100644 --- a/server/pkg/website/storagekeys/tags.go +++ b/server/pkg/website/storagekeys/tags.go @@ -1,5 +1,7 @@ package storagekeys +import "github.com/massalabs/station/pkg/convert" + const ( FILE_TAG = "\x01FILE" FILE_LOCATION_TAG = "\x02LOCATION" @@ -10,3 +12,8 @@ const ( FILE_METADATA_LOCATION_TAG = "\x07FML" DEWEB_VERSION_TAG = "\xFFDEWEB_VERSION" ) + +// FileLocationTag is the tag for the file location as a byte array. +func FileLocationTag() []byte { + return convert.ToBytes(FILE_LOCATION_TAG) +} diff --git a/smart-contract/src/e2e/helpers/serializable/Metadata.ts b/smart-contract/src/e2e/helpers/serializable/Metadata.ts index 5ed373d7..60cc7ebd 100644 --- a/smart-contract/src/e2e/helpers/serializable/Metadata.ts +++ b/smart-contract/src/e2e/helpers/serializable/Metadata.ts @@ -1,10 +1,16 @@ import { Args, DeserializedResult, Serializable } from '@massalabs/massa-web3'; export class Metadata implements Serializable { - constructor(public key: string = '', public value: string = '') {} + constructor( + public key: string = '', + public value: string = '', + ) {} serialize(): Uint8Array { - return new Args().addString(this.key).addString(this.value).serialize(); + return new Args() + .addString(this.key) + .addString(this.value) + .serialize(); } deserialize(data: Uint8Array, offset: number): DeserializedResult {