Add support for custom url generation on s3 stored files

Uses two config keys to support url generation that doesn't directly go
to (compatible s3). Can be used to generate urls to any cache server or
CDN.

config/default.yaml

@@ -100,8 +100,17 @@ s3:
   endpoint: 's3.amazonaws.com' # Will always use https
   videos_bucket: 'videos'
   videos_prefix: '' # Allows setting all buckets to the same value but with a different prefix
+  # Optional, use to overide the default url generation that just gives the public url to s3
+  # Use the string %path% in place where the (full) path to the file should be. The path
+  # does not include the videos_prefix, so you should add it to your template if needed.
+  # For example:
+  #     https://my-videos-cache.example.com/cdn-cache/%path%?thing=true
+  # Would turn into:
+  #     https://my-videos-cache.example.com/cdn-cache/hls/9ffceb57-cbe3-41c5-80e4-dbb7e97c3958/b27d1892-9c5c-4bef-8bce-f68e54fb1208-240-fragmented.mp4?thing=true
+  videos_template: ''
   streaming_playlists_bucket: 'streaming-playlists'
   streaming_playlists_prefix: ''
+  streaming_playlists_template: ''
 
 log:
   level: 'info' # 'debug' | 'info' | 'warn' | 'error'

server/initializers/config.ts

@@ -78,11 +78,13 @@ const CONFIG = {
     ENDPOINT: config.get<string>('s3.endpoint'),
     VIDEOS_BUCKETINFO: {
       bucket: config.get<string>('s3.videos_bucket'),
-      prefix: config.get<string>('s3.videos_prefix')
+      prefix: config.get<string>('s3.videos_prefix'),
+      url_template: config.get<string>('s3.videos_url_template')
     },
     STREAMING_PLAYLISTS_BUCKETINFO: {
       bucket: config.get<string>('s3.streaming_playlists_bucket'),
-      prefix: config.get<string>('s3.streaming_playlists_prefix')
+      prefix: config.get<string>('s3.streaming_playlists_prefix'),
+      url_template: config.get<string>('s3.streaming_playlists_template')
     }
   },
   WEBSERVER: {

server/lib/job-queue/handlers/move-to-object-storage.ts

@@ -4,7 +4,7 @@ import {
   MoveObjectStoragePayload, VideoState
 } from '../../../../shared'
 import { VideoModel } from '@server/models/video/video'
-import { storeObject } from '@server/lib/object-storage'
+import { generateUrl, storeObject } from '@server/lib/object-storage'
 import { CONFIG } from '@server/initializers/config'
 import { join } from 'path'
 import { HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
@@ -54,7 +54,7 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
     )
 
     file.storage = VideoStorageType.OBJECT_STORAGE
-    file.fileUrl = `https://${CONFIG.S3.VIDEOS_BUCKETINFO.bucket}.${CONFIG.S3.ENDPOINT}/${CONFIG.S3.VIDEOS_BUCKETINFO.prefix}${filename}`
+    file.fileUrl = generateUrl(filename, CONFIG.S3.VIDEOS_BUCKETINFO)
     await file.save()
   }
 }
@@ -83,10 +83,8 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
       CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO
     )
 
-    // eslint-disable-next-line max-len
-    playlist.playlistUrl = `https://${CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket}.${CONFIG.S3.ENDPOINT}/${CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO.prefix}${masterPlaylistFilename}`
-    // eslint-disable-next-line max-len
-    playlist.segmentsSha256Url = `https://${CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket}.${CONFIG.S3.ENDPOINT}/${CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO.prefix}${segmentsFileName}`
+    playlist.playlistUrl = generateUrl(masterPlaylistFilename, CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO)
+    playlist.segmentsSha256Url = generateUrl(segmentsFileName, CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO)
 
     for (const videoFile of playlist.VideoFiles) {
       const file = await videoFile.reload()
@@ -114,8 +112,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
 
       // Signals that the video file + playlist file were uploaded
       file.storage = VideoStorageType.OBJECT_STORAGE
-      // eslint-disable-next-line max-len
-      file.fileUrl = `https://${CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket}.${CONFIG.S3.ENDPOINT}/${CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO.prefix}${filename}`
+      file.fileUrl = generateUrl(filename, CONFIG.S3.STREAMING_PLAYLISTS_BUCKETINFO)
       await file.save()
     }
 

server/lib/object-storage.ts

@@ -3,7 +3,7 @@ import { DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObj
 import { CONFIG } from "@server/initializers/config"
 import { logger } from '@server/helpers/logger'
 
-type BucketInfo = {bucket: string, prefix?: string}
+type BucketInfo = {bucket: string, prefix?: string, url_template?: string}
 
 function getS3Client () {
   return new S3Client({ endpoint: `https://${CONFIG.S3.ENDPOINT}` })
@@ -63,3 +63,11 @@ export async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
   // Repeat if not all objects could be listed at once (limit of 1000?)
   if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
 }
+
+export function generateUrl (filename: string, bucketInfo: BucketInfo) {
+  if (!bucketInfo.url_template) {
+    return `https://${bucketInfo.bucket}.${CONFIG.S3.ENDPOINT}/${bucketInfo.prefix}${filename}`
+  }
+  const key = filename
+  return bucketInfo.url_template.replace('%path%', key)
+}