Caption support

go.mod

@@ -47,6 +47,7 @@ require (
 
 require (
 	github.com/apenwarr/fixconsole v0.0.0-20191012055117-5a9f6489cc29
+	github.com/asticode/go-astisub v0.20.0
 	github.com/go-chi/httplog v0.2.1
 	github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4
 	github.com/hashicorp/golang-lru v0.5.4
@@ -61,6 +62,8 @@ require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/apenwarr/w32 v0.0.0-20190407065021-aa00fece76ab // indirect
+	github.com/asticode/go-astikit v0.20.0 // indirect
+	github.com/asticode/go-astits v1.8.0 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

go.sum

@@ -108,6 +108,12 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=
+github.com/asticode/go-astikit v0.20.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0=
+github.com/asticode/go-astisub v0.20.0 h1:mKuLwgGkQj35RRHFiTcq+2hgR7g1mHiYiIkr9UNTmXw=
+github.com/asticode/go-astisub v0.20.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2zLRVFf6bIFQK8=
+github.com/asticode/go-astits v1.8.0 h1:rf6aiiGn/QhlFjNON1n5plqF3Fs025XLUwiQ0NB6oZg=
+github.com/asticode/go-astits v1.8.0/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ=
 github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v1.3.2/go.mod h1:7OaACgj2SX3XGWnrIjGlJM22h6yD6MEWKvm7levnnM8=
 github.com/aws/aws-sdk-go-v2 v1.6.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w=
@@ -629,6 +635,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

graphql/documents/data/config.graphql

@@ -118,6 +118,7 @@ fragment ConfigDefaultSettingsData on ConfigDefaultSettingsResult {
   scan {
     useFileMetadata
     stripFileExtension
+    scanDetectCaptions
     scanGeneratePreviews
     scanGenerateImagePreviews
     scanGenerateSprites

graphql/documents/data/scene-slim.graphql

@@ -13,6 +13,11 @@ fragment SlimSceneData on Scene {
   phash
   interactive
   interactive_speed
+  captions {
+    language_code
+    path
+    caption_type
+  }
 
   file {
     size
@@ -35,6 +40,7 @@ fragment SlimSceneData on Scene {
     sprite
     funscript
     interactive_heatmap
+    caption
   }
 
   scene_markers {

graphql/documents/data/scene.graphql

@@ -13,6 +13,11 @@ fragment SceneData on Scene {
   phash
   interactive
   interactive_speed
+  captions {
+    language_code
+    path
+    caption_type
+  }
   created_at
   updated_at
 
@@ -37,6 +42,7 @@ fragment SceneData on Scene {
     sprite
     funscript
     interactive_heatmap
+    caption
   }
 
   scene_markers {

graphql/schema/types/filters.graphql

@@ -174,6 +174,8 @@ input SceneFilterType {
   interactive: Boolean
   """Filter by InteractiveSpeed"""
   interactive_speed: IntCriterionInput
+  """Filter to only include scenes which have captions. `true` or `false`"""
+  captioned: String
 }
 
 input MovieFilterType {

graphql/schema/types/metadata.graphql

@@ -85,6 +85,8 @@ input ScanMetadataInput {
   scanGeneratePhashes: Boolean
   """Generate image thumbnails during scan"""
   scanGenerateThumbnails: Boolean
+  """Detect scene captions"""
+  scanDetectCaptions: Boolean
 
   "Filter options for the scan"
   filter: ScanMetaDataFilterInput
@@ -105,6 +107,8 @@ type ScanMetadataOptions {
   scanGeneratePhashes: Boolean!
   """Generate image thumbnails during scan"""
   scanGenerateThumbnails: Boolean!
+  """Detect scene captions"""
+  scanDetectCaptions: Boolean!
 }
 
 input CleanMetadataInput {

graphql/schema/types/scene.graphql

@@ -19,13 +19,20 @@ type ScenePathsType {
   sprite: String # Resolver
   funscript: String # Resolver
   interactive_heatmap: String # Resolver
+  caption: String # Resolver
 }
 
 type SceneMovie {
   movie: Movie!
   scene_index: Int
 }
 
+type SceneCaption {
+  language_code: String!
+  path: String!
+  caption_type: String!
+}
+
 type Scene {
   id: ID!
   checksum: String
@@ -41,6 +48,7 @@ type Scene {
   phash: String
   interactive: Boolean!
   interactive_speed: Int
+  captions: [SceneCaption!]
   created_at: Time!
   updated_at: Time!
   file_mod_time: Time

internal/api/resolver_model_scene.go

@@ -98,6 +98,7 @@ func (r *sceneResolver) Paths(ctx context.Context, obj *models.Scene) (*models.S
 	spritePath := builder.GetSpriteURL()
 	chaptersVttPath := builder.GetChaptersVTTURL()
 	funscriptPath := builder.GetFunscriptURL()
+	captionBasePath := builder.GetCaptionURL()
 	interactiveHeatmap := builder.GetInteractiveHeatmapURL()
 
 	return &models.ScenePathsType{
@@ -110,6 +111,7 @@ func (r *sceneResolver) Paths(ctx context.Context, obj *models.Scene) (*models.S
 		Sprite:             &spritePath,
 		Funscript:          &funscriptPath,
 		InteractiveHeatmap: &interactiveHeatmap,
+		Caption:            &captionBasePath,
 	}, nil
 }
 
@@ -124,6 +126,17 @@ func (r *sceneResolver) SceneMarkers(ctx context.Context, obj *models.Scene) (re
 	return ret, nil
 }
 
+func (r *sceneResolver) Captions(ctx context.Context, obj *models.Scene) (ret []*models.SceneCaption, err error) {
+	if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error {
+		ret, err = repo.Scene().GetCaptions(obj.ID)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	return ret, err
+}
+
 func (r *sceneResolver) Galleries(ctx context.Context, obj *models.Scene) (ret []*models.Gallery, err error) {
 	if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error {
 		ret, err = repo.Gallery().FindBySceneID(obj.ID)

internal/api/routes_scene.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"strconv"
@@ -41,6 +42,7 @@ func (rs sceneRoutes) Routes() chi.Router {
 		r.Get("/vtt/chapter", rs.ChapterVtt)
 		r.Get("/funscript", rs.Funscript)
 		r.Get("/interactive_heatmap", rs.InteractiveHeatmap)
+		r.Get("/caption", rs.CaptionLang)
 
 		r.Get("/scene_marker/{sceneMarkerId}/stream", rs.SceneMarkerStream)
 		r.Get("/scene_marker/{sceneMarkerId}/preview", rs.SceneMarkerPreview)
@@ -283,6 +285,49 @@ func (rs sceneRoutes) InteractiveHeatmap(w http.ResponseWriter, r *http.Request)
 	http.ServeFile(w, r, filepath)
 }
 
+func (rs sceneRoutes) Caption(w http.ResponseWriter, r *http.Request, lang string, ext string) {
+	s := r.Context().Value(sceneKey).(*models.Scene)
+
+	path := ""
+	if err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error {
+		var err error
+		captions, err := repo.Scene().GetCaptions(s.ID)
+		for _, caption := range captions {
+			if lang == caption.LanguageCode && ext == caption.CaptionType {
+				path = caption.Path
+
+				sub, err := scene.ReadSubs(path)
+				if err == nil {
+					var b bytes.Buffer
+					err = sub.WriteToWebVTT(&b)
+					if err == nil {
+						w.Header().Set("Content-Type", "text/vtt")
+						w.Header().Add("Cache-Control", "no-cache")
+						_, _ = b.WriteTo(w)
+					}
+					return err
+				}
+				logger.Debugf("Error while reading subs: %v", err)
+			}
+		}
+		return err
+	}); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func (rs sceneRoutes) CaptionLang(w http.ResponseWriter, r *http.Request) {
+	// serve caption based on lang query param, if provided
+	if err := r.ParseForm(); err != nil {
+		logger.Warnf("[caption] error parsing query form: %v", err)
+	}
+
+	l := r.Form.Get("lang")
+	ext := r.Form.Get("type")
+	rs.Caption(w, r, l, ext)
+}
+
 func (rs sceneRoutes) VttThumbs(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 	w.Header().Set("Content-Type", "text/vtt")

internal/api/urlbuilders/scene.go

@@ -67,6 +67,10 @@ func (b SceneURLBuilder) GetFunscriptURL() string {
 	return b.BaseURL + "/scene/" + b.SceneID + "/funscript"
 }
 
+func (b SceneURLBuilder) GetCaptionURL() string {
+	return b.BaseURL + "/scene/" + b.SceneID + "/caption"
+}
+
 func (b SceneURLBuilder) GetInteractiveHeatmapURL() string {
 	return b.BaseURL + "/scene/" + b.SceneID + "/interactive_heatmap"
 }

internal/manager/manager_tasks.go

@@ -12,13 +12,18 @@ import (
 	"github.com/stashapp/stash/pkg/job"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
 )
 
 func isGallery(pathname string) bool {
 	gExt := config.GetInstance().GetGalleryExtensions()
 	return fsutil.MatchExtension(pathname, gExt)
 }
 
+func isCaptions(pathname string) bool {
+	return fsutil.MatchExtension(pathname, scene.CaptionExts)
+}
+
 func isVideo(pathname string) bool {
 	vidExt := config.GetInstance().GetVideoExtensions()
 	return fsutil.MatchExtension(pathname, vidExt)

internal/manager/task_scan.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stashapp/stash/pkg/job"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/scene/generate"
 	"github.com/stashapp/stash/pkg/utils"
 )
@@ -89,6 +90,7 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
 			file:                 file.FSFile(f.path, f.info),
 			UseFileMetadata:      utils.IsTrue(input.UseFileMetadata),
 			StripFileExtension:   utils.IsTrue(input.StripFileExtension),
+			DetectCaptions:       utils.IsTrue(input.ScanDetectCaptions),
 			fileNamingAlgorithm:  fileNamingAlgo,
 			calculateMD5:         calculateMD5,
 			GeneratePreview:      utils.IsTrue(input.ScanGeneratePreviews),
@@ -252,6 +254,7 @@ type ScanTask struct {
 	file                 file.SourceFile
 	UseFileMetadata      bool
 	StripFileExtension   bool
+	DetectCaptions       bool
 	calculateMD5         bool
 	fileNamingAlgorithm  models.HashAlgorithm
 	GenerateSprite       bool
@@ -277,6 +280,8 @@ func (t *ScanTask) Start(ctx context.Context) {
 			s = t.scanScene(ctx)
 		case isImage(path):
 			t.scanImage(ctx)
+		case t.DetectCaptions && isCaptions(path):
+			t.associateCaptions(ctx)
 		}
 	})
 
@@ -351,6 +356,7 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error {
 	vidExt := config.GetVideoExtensions()
 	imgExt := config.GetImageExtensions()
 	gExt := config.GetGalleryExtensions()
+	capExt := scene.CaptionExts
 	excludeVidRegex := generateRegexps(config.GetExcludes())
 	excludeImgRegex := generateRegexps(config.GetImageExcludes())
 
@@ -394,6 +400,10 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error {
 			}
 		}
 
+		if fsutil.MatchExtension(path, capExt) {
+			return f(path, info, err)
+		}
+
 		return nil
 	})
 }

internal/manager/task_scan_scene.go

@@ -2,7 +2,9 @@ package manager
 
 import (
 	"context"
+	"path/filepath"
 
+	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/pkg/ffmpeg"
 	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/logger"
@@ -60,6 +62,7 @@ func (t *ScanTask) scanScene(ctx context.Context) *models.Scene {
 		PluginCache:      instance.PluginCache,
 		MutexManager:     t.mutexManager,
 		UseFileMetadata:  t.UseFileMetadata,
+		DetectCaptions:   t.DetectCaptions,
 	}
 
 	if s != nil {
@@ -78,3 +81,48 @@ func (t *ScanTask) scanScene(ctx context.Context) *models.Scene {
 
 	return retScene
 }
+
+// associates captions to scene/s with the same basename
+func (t *ScanTask) associateCaptions(ctx context.Context) {
+	vExt := config.GetInstance().GetVideoExtensions()
+	captionPath := t.file.Path()
+	captionLang := scene.GetCaptionsLangFromPath(captionPath)
+
+	relatedFiles := scene.GenerateCaptionCandidates(captionPath, vExt)
+	if err := t.TxnManager.WithTxn(ctx, func(r models.Repository) error {
+		var err error
+		sqb := r.Scene()
+
+		for _, scenePath := range relatedFiles {
+			s, er := sqb.FindByPath(scenePath)
+
+			if er != nil {
+				logger.Errorf("Error searching for scene %s: %v", scenePath, er)
+				continue
+			}
+			if s != nil { // found related Scene
+				logger.Debugf("Matched captions to scene %s", s.Path)
+				captions, er := sqb.GetCaptions(s.ID)
+				if er == nil {
+					fileExt := filepath.Ext(captionPath)
+					ext := fileExt[1:]
+					if !scene.IsLangInCaptions(captionLang, ext, captions) { // only update captions if language code is not present
+						newCaption := &models.SceneCaption{
+							LanguageCode: captionLang,
+							Path:         captionPath,
+							CaptionType:  ext,
+						}
+						captions = append(captions, newCaption)
+						er = sqb.UpdateCaptions(s.ID, captions)
+						if er == nil {
+							logger.Debugf("Updated captions for scene %s. Added %s", s.Path, captionLang)
+						}
+					}
+				}
+			}
+		}
+		return err
+	}); err != nil {
+		logger.Error(err.Error())
+	}
+}

pkg/database/database.go

@@ -23,7 +23,7 @@ import (
 var DB *sqlx.DB
 var WriteMu sync.Mutex
 var dbPath string
-var appSchemaVersion uint = 30
+var appSchemaVersion uint = 31
 var databaseSchemaVersion uint
 
 //go:embed migrations/*.sql