From bd2879449cc843360c19ad3e6aa2e3b3d318c8c1 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Tue, 6 Jul 2021 07:52:20 +0200 Subject: [PATCH 1/3] video upload: set timeout to 15 seconds --- server/controllers/api/videos/upload.ts | 108 +++++++++++++++++++----- server/typings/express/index.d.ts | 3 + 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index bcd21ac9987..d0e3894bf80 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -87,6 +87,7 @@ uploadRouter.put('/upload-resumable', authenticate, uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes asyncMiddleware(videosAddResumableValidator), + asyncMiddleware(moveVideoFileResumable()), asyncMiddleware(addVideoResumable) ) @@ -113,7 +114,9 @@ export async function addVideoLegacy (req: express.Request, res: express.Respons const videoInfo: VideoCreate = req.body const files = req.files - return addVideo({ res, videoPhysicalFile, videoInfo, files }) + await moveVideoFile(res, videoPhysicalFile) + + return addVideo({ res, videoInfo, files }) } export async function addVideoResumable (_req: express.Request, res: express.Response) { @@ -124,7 +127,7 @@ export async function addVideoResumable (_req: express.Request, res: express.Res // Don't need the meta file anymore await deleteResumableUploadMetaFile(videoPhysicalFile.path) - return addVideo({ res, videoPhysicalFile, videoInfo, files }) + return addVideo({ res, videoInfo, files }) } async function addVideo (options: { @@ -134,26 +137,11 @@ async function addVideo (options: { files: express.UploadFiles }) { const { res, videoPhysicalFile, videoInfo, files } = options - const videoChannel = res.locals.videoChannel + const video = res.locals.video + const videoFile = res.locals.videoFile const user = res.locals.oauth.token.User - - const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) - - videoData.state = CONFIG.TRANSCODING.ENABLED - ? VideoState.TO_TRANSCODE - : VideoState.PUBLISHED - - videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware - - const video = new VideoModel(videoData) as MVideoFullLight - video.VideoChannel = videoChannel - video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object - - const videoFile = await buildNewFile(video, videoPhysicalFile) - - // Move physical file const destination = getVideoFilePath(video, videoFile) - await move(videoPhysicalFile.path, destination) + // This is important in case if there is another attempt in the retry process videoPhysicalFile.filename = getVideoFilePath(video, videoFile) videoPhysicalFile.path = destination @@ -225,6 +213,86 @@ async function addVideo (options: { }) } +function moveVideoFileResumable () { + interface FileMoveQueueItem { + status: 'moving' | 'done' + video: MVideoFullLight + videoFile: VideoFileModel + } + const fileMoveInProgress: { [key: string]: FileMoveQueueItem } | {} = {} + + return async function (req: express.Request, res: express.Response, next: express.NextFunction) { + const uploadId = req.body.id + + const moveProcess: Promise<'timeout' | 'done'> = new Promise(async resolve => { + fileMoveInProgress[uploadId] = fileMoveInProgress[uploadId] || {} + + if (fileMoveInProgress[uploadId].status === 'moving') { + return + } + + if (fileMoveInProgress[uploadId].status === undefined) { + fileMoveInProgress[uploadId] = { + status: 'moving' + } + + const { video, videoFile } = await moveVideoFile(res, res.locals.videoFileResumable) + + fileMoveInProgress[uploadId] = { + status: 'done', + videoFile, + video + } + } + + resolve('done') + }) + + const status = await Promise.race([ + moveProcess, + new Promise(resolve => setTimeout(() => resolve('timeout'), 1000 * 15)) + ]) + + if (status === 'timeout') { + return res.status(308).json({}) + } else { + res.locals.video = fileMoveInProgress[uploadId].video + res.locals.videoFile = fileMoveInProgress[uploadId].videoFile + + // uploadId is a hash of file metadata, so we need to reset this to handle a second upload with the same file + fileMoveInProgress[uploadId] = undefined + + return next() + } + } +} + +async function moveVideoFile (res: express.Response, videoPhysicalFile: express.EnhancedUploadXFile) { + // const videoPhysicalFile = res.locals.videoFileResumable + const videoInfo = videoPhysicalFile.metadata + const videoChannel = res.locals.videoChannel + const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) + videoData.state = CONFIG.TRANSCODING.ENABLED + ? VideoState.TO_TRANSCODE + : VideoState.PUBLISHED + + videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware + const video = new VideoModel(videoData) as MVideoFullLight + video.VideoChannel = videoChannel + video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object + + const videoFile = await buildNewFile(video, videoPhysicalFile) + + // Move physical file + const destination = getVideoFilePath(video, videoFile) + await move(videoPhysicalFile.path, destination) + + return { + video, + videoFile + } +} + async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { const videoFile = new VideoFileModel({ extname: getLowercaseExtension(videoPhysicalFile.filename), diff --git a/server/typings/express/index.d.ts b/server/typings/express/index.d.ts index 1a8dc343097..df7d0b141a7 100644 --- a/server/typings/express/index.d.ts +++ b/server/typings/express/index.d.ts @@ -105,6 +105,9 @@ declare module 'express' { videoAll?: MVideoFullLight onlyImmutableVideo?: MVideoImmutable onlyVideo?: MVideoThumbnail + + video?: MVideoFullLight + videoId?: MVideoId videoLive?: MVideoLive From cf9ac8fb93fa3e8dd018fdeff110e1da7803fccf Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Wed, 7 Jul 2021 16:22:18 +0200 Subject: [PATCH 2/3] refactor(server/upload): remove unused code --- server/controllers/api/videos/upload.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index d0e3894bf80..7b97afe4dee 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -132,19 +132,13 @@ export async function addVideoResumable (_req: express.Request, res: express.Res async function addVideo (options: { res: express.Response - videoPhysicalFile: express.VideoUploadFile videoInfo: VideoCreate files: express.UploadFiles }) { - const { res, videoPhysicalFile, videoInfo, files } = options + const { res, videoInfo, files } = options const video = res.locals.video const videoFile = res.locals.videoFile const user = res.locals.oauth.token.User - const destination = getVideoFilePath(video, videoFile) - - // This is important in case if there is another attempt in the retry process - videoPhysicalFile.filename = getVideoFilePath(video, videoFile) - videoPhysicalFile.path = destination const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ video, From b6943536ab9b231a1cd1e1233d3dd3ee8cad7342 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Tue, 27 Jul 2021 12:05:30 +0200 Subject: [PATCH 3/3] WIP: handle video file move in a job --- server/controllers/api/videos/live.ts | 1 + server/controllers/api/videos/update.ts | 4 +- server/controllers/api/videos/upload.ts | 221 +++--------------- server/initializers/constants.ts | 12 +- .../job-queue/handlers/video-file-process.ts | 170 ++++++++++++++ server/lib/job-queue/job-queue.ts | 5 + server/lib/video.ts | 8 +- .../middlewares/validators/videos/videos.ts | 1 + server/models/video/video.ts | 5 +- shared/models/server/job.model.ts | 10 + shared/models/videos/video-state.enum.ts | 3 +- 11 files changed, 236 insertions(+), 204 deletions(-) create mode 100644 server/lib/job-queue/handlers/video-file-process.ts diff --git a/server/controllers/api/videos/live.ts b/server/controllers/api/videos/live.ts index d8c51c2d41b..8dbc9d2fd23 100644 --- a/server/controllers/api/videos/live.ts +++ b/server/controllers/api/videos/live.ts @@ -98,6 +98,7 @@ async function addLiveVideo (req: express.Request, res: express.Response) { const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ video, + // @ts-expect-error files: req.files, fallback: type => { return updateVideoMiniatureFromExisting({ diff --git a/server/controllers/api/videos/update.ts b/server/controllers/api/videos/update.ts index 8affe71c682..e2ad3e1c5f8 100644 --- a/server/controllers/api/videos/update.ts +++ b/server/controllers/api/videos/update.ts @@ -60,9 +60,11 @@ export async function updateVideo (req: express.Request, res: express.Response) const wasConfidentialVideo = videoInstance.isConfidential() const hadPrivacyForFederation = videoInstance.hasPrivacyForFederation() + // @ts-expect-error + const files: { [fieldname: string]: Express.Multer.File[] } = req.files const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ video: videoInstance, - files: req.files, + files, fallback: () => Promise.resolve(undefined), automaticallyGenerated: false }) diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 7b97afe4dee..288f91418ef 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -1,30 +1,18 @@ import * as express from 'express' -import { move } from 'fs-extra' -import { getLowercaseExtension } from '@server/helpers/core-utils' import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' import { uuidToShort } from '@server/helpers/uuid' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' -import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' -import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { buildLocalVideoFromReq, setVideoTags } from '@server/lib/video' import { openapiOperationDoc } from '@server/middlewares/doc' -import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { MVideoFullLight } from '@server/types/models' import { uploadx } from '@uploadx/core' import { VideoCreate, VideoState } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/core-utils/miscs' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' -import { retryTransactionWrapper } from '../../../helpers/database-utils' import { createReqFiles } from '../../../helpers/express-utils' -import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' import { logger, loggerTagsFactory } from '../../../helpers/logger' import { CONFIG } from '../../../initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, MIMETYPES } from '../../../initializers/constants' +import { MIMETYPES } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' -import { federateVideoIfNeeded } from '../../../lib/activitypub/videos' -import { Notifier } from '../../../lib/notifier' -import { Hooks } from '../../../lib/plugins/hooks' -import { generateVideoMiniature } from '../../../lib/thumbnail' -import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' import { asyncMiddleware, asyncRetryTransactionMiddleware, @@ -34,8 +22,8 @@ import { videosAddResumableValidator } from '../../../middlewares' import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' -import { VideoModel } from '../../../models/video/video' -import { VideoFileModel } from '../../../models/video/video-file' +import { JobQueue } from '@server/lib/job-queue' +import { VideoModel } from '@server/models/video/video' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') @@ -87,7 +75,6 @@ uploadRouter.put('/upload-resumable', authenticate, uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes asyncMiddleware(videosAddResumableValidator), - asyncMiddleware(moveVideoFileResumable()), asyncMiddleware(addVideoResumable) ) @@ -112,56 +99,44 @@ export async function addVideoLegacy (req: express.Request, res: express.Respons const videoPhysicalFile = req.files['videofile'][0] const videoInfo: VideoCreate = req.body - const files = req.files - await moveVideoFile(res, videoPhysicalFile) - - return addVideo({ res, videoInfo, files }) + return addVideo({ res, videoInfo, videoPhysicalFile }) } export async function addVideoResumable (_req: express.Request, res: express.Response) { const videoPhysicalFile = res.locals.videoFileResumable const videoInfo = videoPhysicalFile.metadata - const files = { previewfile: videoInfo.previewfile } // Don't need the meta file anymore await deleteResumableUploadMetaFile(videoPhysicalFile.path) - return addVideo({ res, videoInfo, files }) + return addVideo({ res, videoInfo, videoPhysicalFile }) } async function addVideo (options: { res: express.Response videoInfo: VideoCreate - files: express.UploadFiles + videoPhysicalFile: express.VideoUploadFile }) { - const { res, videoInfo, files } = options - const video = res.locals.video - const videoFile = res.locals.videoFile + const { res, videoInfo, videoPhysicalFile } = options + const files = { previewfile: videoInfo.previewfile } + const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) + const video = new VideoModel(videoData) as MVideoFullLight + video.state = VideoState.TO_PROCESS + video.duration = 0 + video.url = '' + video.VideoFiles = [] + // const videoFile = res.locals.videoFile const user = res.locals.oauth.token.User - const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ - video, - files, - fallback: type => generateVideoMiniature({ video, videoFile, type }) - }) - const { videoCreated } = await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight - await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - await videoCreated.addAndSaveThumbnail(previewModel, t) - // Do not forget to add video channel information to the created video videoCreated.VideoChannel = res.locals.videoChannel - videoFile.videoId = video.id - await videoFile.save(sequelizeOptions) - - video.VideoFiles = [ videoFile ] - await setVideoTags({ video, tags: videoInfo.tags, transaction: t }) // Schedule an update in the future? @@ -173,31 +148,23 @@ async function addVideo (options: { }, sequelizeOptions) } - // Channel has a new content, set as updated - await videoCreated.VideoChannel.setAsUpdated(t) - - await autoBlacklistVideoIfNeeded({ - video, - user, - isRemote: false, - isNew: true, - transaction: t - }) - auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid)) + JobQueue.Instance.createJob({ + type: 'video-process', + payload: { + previewFilePath: files.previewfile?.[0].path, + videoFilePath: res.locals.videoFileResumable.path, + videoId: videoCreated.id, + videoPhysicalFile, + userId: user.id + } + }) + return { videoCreated } }) - createTorrentFederate(video, videoFile) - - if (video.state === VideoState.TO_TRANSCODE) { - await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) - } - - Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) - return res.json({ video: { id: videoCreated.id, @@ -206,135 +173,3 @@ async function addVideo (options: { } }) } - -function moveVideoFileResumable () { - interface FileMoveQueueItem { - status: 'moving' | 'done' - video: MVideoFullLight - videoFile: VideoFileModel - } - const fileMoveInProgress: { [key: string]: FileMoveQueueItem } | {} = {} - - return async function (req: express.Request, res: express.Response, next: express.NextFunction) { - const uploadId = req.body.id - - const moveProcess: Promise<'timeout' | 'done'> = new Promise(async resolve => { - fileMoveInProgress[uploadId] = fileMoveInProgress[uploadId] || {} - - if (fileMoveInProgress[uploadId].status === 'moving') { - return - } - - if (fileMoveInProgress[uploadId].status === undefined) { - fileMoveInProgress[uploadId] = { - status: 'moving' - } - - const { video, videoFile } = await moveVideoFile(res, res.locals.videoFileResumable) - - fileMoveInProgress[uploadId] = { - status: 'done', - videoFile, - video - } - } - - resolve('done') - }) - - const status = await Promise.race([ - moveProcess, - new Promise(resolve => setTimeout(() => resolve('timeout'), 1000 * 15)) - ]) - - if (status === 'timeout') { - return res.status(308).json({}) - } else { - res.locals.video = fileMoveInProgress[uploadId].video - res.locals.videoFile = fileMoveInProgress[uploadId].videoFile - - // uploadId is a hash of file metadata, so we need to reset this to handle a second upload with the same file - fileMoveInProgress[uploadId] = undefined - - return next() - } - } -} - -async function moveVideoFile (res: express.Response, videoPhysicalFile: express.EnhancedUploadXFile) { - // const videoPhysicalFile = res.locals.videoFileResumable - const videoInfo = videoPhysicalFile.metadata - const videoChannel = res.locals.videoChannel - const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) - videoData.state = CONFIG.TRANSCODING.ENABLED - ? VideoState.TO_TRANSCODE - : VideoState.PUBLISHED - - videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware - const video = new VideoModel(videoData) as MVideoFullLight - video.VideoChannel = videoChannel - video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object - - const videoFile = await buildNewFile(video, videoPhysicalFile) - - // Move physical file - const destination = getVideoFilePath(video, videoFile) - await move(videoPhysicalFile.path, destination) - - return { - video, - videoFile - } -} - -async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { - const videoFile = new VideoFileModel({ - extname: getLowercaseExtension(videoPhysicalFile.filename), - size: videoPhysicalFile.size, - videoStreamingPlaylistId: null, - metadata: await getMetadataFromFile(videoPhysicalFile.path) - }) - - if (videoFile.isAudio()) { - videoFile.resolution = DEFAULT_AUDIO_RESOLUTION - } else { - videoFile.fps = await getVideoFileFPS(videoPhysicalFile.path) - videoFile.resolution = (await getVideoFileResolution(videoPhysicalFile.path)).videoFileResolution - } - - videoFile.filename = generateVideoFilename(video, false, videoFile.resolution, videoFile.extname) - - return videoFile -} - -async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoFile) { - await createTorrentAndSetInfoHash(video, fileArg) - - // Refresh videoFile because the createTorrentAndSetInfoHash could be long - const refreshedFile = await VideoFileModel.loadWithVideo(fileArg.id) - // File does not exist anymore, remove the generated torrent - if (!refreshedFile) return fileArg.removeTorrent() - - refreshedFile.infoHash = fileArg.infoHash - refreshedFile.torrentFilename = fileArg.torrentFilename - - return refreshedFile.save() -} - -function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void { - // Create the torrent file in async way because it could be long - createTorrentAndSetInfoHashAsync(video, videoFile) - .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) })) - .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)) - .then(refreshedVideo => { - if (!refreshedVideo) return - - // Only federate and notify after the torrent creation - Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) - - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t)) - }) - }) - .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) })) -} diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index cd00b73d596..4b2bf9d5400 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { 'videos-views': 1, 'activitypub-refresher': 1, 'video-redundancy': 1, - 'video-live-ending': 1 + 'video-live-ending': 1, + 'video-process': 1 } // Excluded keys are jobs that can be configured by admins const JOB_CONCURRENCY: { [id in Exclude]: number } = { @@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude generateVideoMiniature({ video, videoFile, type }) + }) + + await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + await video.addAndSaveThumbnail(thumbnailModel, t) + await video.addAndSaveThumbnail(previewModel, t) + video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object + + video.state = CONFIG.TRANSCODING.ENABLED + ? VideoState.TO_TRANSCODE + : VideoState.PUBLISHED + + videoFile.videoId = video.id + await videoFile.save(sequelizeOptions) + + video.VideoFiles = [ videoFile ] + + // Channel has a new content, set as updated + await video.VideoChannel.setAsUpdated(t) + + await autoBlacklistVideoIfNeeded({ + video, + user, + isRemote: false, + isNew: true, + transaction: t + }) + }) + + createTorrentFederate(video, videoFile) + + if (video.state === VideoState.TO_TRANSCODE) { + await addOptimizeOrMergeAudioJob(video, videoFile, user) + } + + Hooks.runAction('action:api.video.uploaded', { video: video }) +} + +function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void { + // Create the torrent file in async way because it could be long + createTorrentAndSetInfoHashAsync(video, videoFile) + .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err })) + .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)) + .then(refreshedVideo => { + if (!refreshedVideo) return + + // Only federate and notify after the torrent creation + Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) + + return retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t)) + }) + }) + .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err })) +} + +async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoFile) { + await createTorrentAndSetInfoHash(video, fileArg) + + // Refresh videoFile because the createTorrentAndSetInfoHash could be long + const refreshedFile = await VideoFileModel.loadWithVideo(fileArg.id) + // File does not exist anymore, remove the generated torrent + if (!refreshedFile) return fileArg.removeTorrent() + + refreshedFile.infoHash = fileArg.infoHash + refreshedFile.torrentFilename = fileArg.torrentFilename + + return refreshedFile.save() +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 42e8347b1f6..a3ee808e972 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -34,6 +34,8 @@ import { processVideoImport } from './handlers/video-import' import { processVideoLiveEnding } from './handlers/video-live-ending' import { processVideoTranscoding } from './handlers/video-transcoding' import { processVideosViews } from './handlers/video-views' +import { processVideoProcess } from './handlers/video-file-process' +import { VideoProcessPayload } from '../../../shared/models/server/job.model' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -45,6 +47,7 @@ type CreateJobArgument = { type: 'video-transcoding', payload: VideoTranscodingPayload } | { type: 'email', payload: EmailPayload } | { type: 'video-import', payload: VideoImportPayload } | + { type: 'video-process', payload: VideoProcessPayload } | { type: 'activitypub-refresher', payload: RefreshPayload } | { type: 'videos-views', payload: {} } | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | @@ -62,6 +65,7 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise } = { 'activitypub-http-fetcher': processActivityPubHttpFetcher, 'activitypub-cleaner': processActivityPubCleaner, 'activitypub-follow': processActivityPubFollow, + 'video-process': processVideoProcess, 'video-file-import': processVideoFileImport, 'video-transcoding': processVideoTranscoding, 'email': processEmail, @@ -83,6 +87,7 @@ const jobTypes: JobType[] = [ 'video-transcoding', 'video-file-import', 'video-import', + 'video-process', 'videos-views', 'activitypub-refresher', 'video-redundancy', diff --git a/server/lib/video.ts b/server/lib/video.ts index daf998704b5..6d871a307da 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -1,4 +1,3 @@ -import { UploadFiles } from 'express' import { Transaction } from 'sequelize/types' import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' import { sequelizeTypescript } from '@server/initializers/database' @@ -35,7 +34,11 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil async function buildVideoThumbnailsFromReq (options: { video: MVideoThumbnail - files: UploadFiles + files: { + [fieldname: string]: { + path: string + }[] + } fallback: (type: ThumbnailType) => Promise automaticallyGenerated?: boolean }) { @@ -52,6 +55,7 @@ async function buildVideoThumbnailsFromReq (options: { } ].map(p => { const fields = files?.[p.fieldName] + console.log(files, fields) if (fields) { return updateVideoMiniatureFromExisting({ diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index 49e10e2b549..322f78b3205 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -601,6 +601,7 @@ export async function isVideoAccepted ( return true } +// Remove!!!! async function addDurationToVideo (videoFile: { path: string, duration?: number }) { const duration: number = await getDurationFromVideoFile(videoFile.path) diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 1e5648a36cb..63d212b07eb 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -40,7 +40,6 @@ import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { VideoFilter } from '../../../shared/models/videos/video-query.type' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { peertubeTruncate } from '../../helpers/core-utils' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { isBooleanValid } from '../../helpers/custom-validators/misc' import { isVideoDescriptionValid, @@ -525,8 +524,8 @@ export class VideoModel extends Model>> { @Column isLive: boolean - @AllowNull(false) - @Is('VideoUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @AllowNull(true) + // @Is('VideoUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) url: string diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 4ab249e0b86..9b41f20d204 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -1,6 +1,7 @@ import { ContextType } from '../activitypub/context' import { VideoResolution } from '../videos/video-resolution.enum' import { SendEmailOptions } from './emailer.model' +import express = require('express') export type JobState = 'active' | 'completed' | 'failed' | 'waiting' | 'delayed' | 'paused' @@ -14,6 +15,7 @@ export type JobType = | 'video-transcoding' | 'email' | 'video-import' + | 'video-process' | 'videos-views' | 'activitypub-refresher' | 'video-redundancy' @@ -89,6 +91,14 @@ export type VideoImportTorrentPayload = { } export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload +export type VideoProcessPayload = { + previewFilePath: string + videoFilePath: string + videoId: number + videoPhysicalFile: express.VideoUploadFile + userId: number +} + export type VideoRedundancyPayload = { videoId: number } diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts index 49d997f240d..3b88c6a568f 100644 --- a/shared/models/videos/video-state.enum.ts +++ b/shared/models/videos/video-state.enum.ts @@ -3,5 +3,6 @@ export const enum VideoState { TO_TRANSCODE = 2, TO_IMPORT = 3, WAITING_FOR_LIVE = 4, - LIVE_ENDED = 5 + LIVE_ENDED = 5, + TO_PROCESS = 6 }