From e08a22c5bf807b43226b92d194209f0e939b6087 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sat, 11 Nov 2023 22:25:20 +0100 Subject: [PATCH 01/20] migrate bookmark content route to new http server --- internal/cmd/add.go | 2 +- internal/cmd/check.go | 2 +- internal/cmd/import.go | 4 +- internal/cmd/pocket.go | 4 +- internal/cmd/root.go | 6 +- internal/cmd/server.go | 5 +- internal/cmd/update.go | 2 +- internal/cmd/utils.go | 2 +- internal/core/ebook.go | 2 +- internal/core/ebook_test.go | 14 +- internal/core/processing.go | 4 +- internal/core/processing_test.go | 14 +- internal/database/database.go | 6 +- internal/database/database_test.go | 30 +-- internal/database/mysql.go | 12 +- internal/database/pg.go | 14 +- internal/database/sqlite.go | 12 +- internal/database/sqlite_test.go | 4 +- .../{config => dependencies}/dependencies.go | 12 +- internal/domains/archiver.go | 13 +- internal/domains/bookmarks.go | 133 +++++++++++++ internal/domains/filesystem.go | 24 +++ internal/domains/utils.go | 46 +++++ internal/http/middleware/auth.go | 4 +- internal/http/response/file.go | 23 +++ internal/http/routes/api/v1/api.go | 6 +- internal/http/routes/api/v1/auth.go | 6 +- internal/http/routes/api/v1/bookmarks.go | 11 +- internal/http/routes/api/v1/tags.go | 6 +- internal/http/routes/bookmark.go | 170 +++++++++++++---- internal/http/routes/frontend.go | 15 -- internal/http/routes/frontend_test.go | 3 + internal/http/routes/legacy.go | 13 +- internal/http/server.go | 11 +- internal/http/templates/templates.go | 25 +++ internal/model/account.go | 34 ++-- internal/model/{model.go => bookmark.go} | 15 +- internal/model/db.go | 3 + internal/model/tag.go | 9 + internal/testutil/shiori.go | 5 +- internal/view/archive.html | 20 ++ internal/view/content.html | 2 +- internal/webserver/handler-api-ext.go | 4 +- internal/webserver/handler-api.go | 6 +- internal/webserver/handler-ui.go | 177 ------------------ internal/webserver/handler.go | 4 +- internal/webserver/server.go | 4 +- 47 files changed, 575 insertions(+), 368 deletions(-) rename internal/{config => dependencies}/dependencies.go (52%) create mode 100644 internal/domains/bookmarks.go create mode 100644 internal/domains/filesystem.go create mode 100644 internal/domains/utils.go create mode 100644 internal/http/response/file.go create mode 100644 internal/http/templates/templates.go rename internal/model/{model.go => bookmark.go} (73%) create mode 100644 internal/model/db.go create mode 100644 internal/model/tag.go create mode 100644 internal/view/archive.html diff --git a/internal/cmd/add.go b/internal/cmd/add.go index 2c20fe34e..e77b3787f 100644 --- a/internal/cmd/add.go +++ b/internal/cmd/add.go @@ -45,7 +45,7 @@ func addHandler(cmd *cobra.Command, args []string) { excerpt = normalizeSpace(excerpt) // Create bookmark item - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: url, Title: title, Excerpt: excerpt, diff --git a/internal/cmd/check.go b/internal/cmd/check.go index 702dc632f..0f41f3f35 100644 --- a/internal/cmd/check.go +++ b/internal/cmd/check.go @@ -76,7 +76,7 @@ func checkHandler(cmd *cobra.Command, args []string) { for i, book := range bookmarks { wg.Add(1) - go func(i int, book model.Bookmark) { + go func(i int, book model.BookmarkDTO) { // Make sure to finish the WG defer wg.Done() diff --git a/internal/cmd/import.go b/internal/cmd/import.go index 48665fc04..3f0fa3ca9 100644 --- a/internal/cmd/import.go +++ b/internal/cmd/import.go @@ -52,7 +52,7 @@ func importHandler(cmd *cobra.Command, args []string) { defer srcFile.Close() // Parse bookmark's file - bookmarks := []model.Bookmark{} + bookmarks := []model.BookmarkDTO{} mapURL := make(map[string]struct{}) doc, err := goquery.NewDocumentFromReader(srcFile) @@ -135,7 +135,7 @@ func importHandler(cmd *cobra.Command, args []string) { } // Add item to list - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ URL: url, Title: title, Tags: tags, diff --git a/internal/cmd/pocket.go b/internal/cmd/pocket.go index a16ad0e9b..1f1278c39 100644 --- a/internal/cmd/pocket.go +++ b/internal/cmd/pocket.go @@ -36,7 +36,7 @@ func pocketHandler(cmd *cobra.Command, args []string) { defer srcFile.Close() // Parse pocket's file - bookmarks := []model.Bookmark{} + bookmarks := []model.BookmarkDTO{} mapURL := make(map[string]struct{}) doc, err := goquery.NewDocumentFromReader(srcFile) @@ -93,7 +93,7 @@ func pocketHandler(cmd *cobra.Command, args []string) { } // Add item to list - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ URL: url, Title: title, Modified: modified.Format(model.DatabaseDateFormat), diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 1af24fb9b..a2e86b3e8 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -8,6 +8,7 @@ import ( "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/domains" "github.com/go-shiori/shiori/internal/model" "github.com/sirupsen/logrus" @@ -47,7 +48,7 @@ func ShioriCmd() *cobra.Command { return rootCmd } -func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *config.Dependencies) { +func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *dependencies.Dependencies) { logger := logrus.New() portableMode, _ := cmd.Flags().GetBool("portable") @@ -96,9 +97,10 @@ func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *confi logger.Warn("Development mode is ENABLED, this will enable some helpers for local development, unsuitable for production environments") } - dependencies := config.NewDependencies(logger, db, cfg) + dependencies := dependencies.NewDependencies(logger, db, cfg) dependencies.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db) dependencies.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir) + dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(logger, db, cfg) // Workaround: Get accounts to make sure at least one is present in the database. // If there's no accounts in the database, create the shiori/gopher account the legacy api diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 68c1809a1..53818ac8c 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -66,7 +66,10 @@ func newServerCommandHandler() func(cmd *cobra.Command, args []string) { dependencies.Log.Infof("Starting Shiori v%s", model.BuildVersion) - server := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies) + server, err := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies) + if err != nil { + dependencies.Log.WithError(err).Fatal("error setting up server") + } if err := server.Start(ctx); err != nil { dependencies.Log.WithError(err).Fatal("error starting server") diff --git a/internal/cmd/update.go b/internal/cmd/update.go index 09c1d32a4..e0f8938bf 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -147,7 +147,7 @@ func updateHandler(cmd *cobra.Command, args []string) { book.URL = url } - go func(i int, book model.Bookmark) { + go func(i int, book model.BookmarkDTO) { // Make sure to finish the WG defer wg.Done() diff --git a/internal/cmd/utils.go b/internal/cmd/utils.go index 764d5b776..fff199b1c 100644 --- a/internal/cmd/utils.go +++ b/internal/cmd/utils.go @@ -41,7 +41,7 @@ func isURLValid(s string) bool { return err == nil && tmp.Scheme != "" && tmp.Hostname() != "" } -func printBookmarks(bookmarks ...model.Bookmark) { +func printBookmarks(bookmarks ...model.BookmarkDTO) { for _, bookmark := range bookmarks { // Create bookmark index strBookmarkIndex := fmt.Sprintf("%d. ", bookmark.ID) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index d3720cce1..360cb0982 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -15,7 +15,7 @@ import ( // GenerateEbook receives a `ProcessRequest` and generates an ebook file in the destination path specified. // The destination path `dstPath` should include file name with ".epub" extension // The bookmark model will be used to update the UI based on whether this function is successful or not. -func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) { +func GenerateEbook(req ProcessRequest, dstPath string) (book model.BookmarkDTO, err error) { book = req.Bookmark diff --git a/internal/core/ebook_test.go b/internal/core/ebook_test.go index 4bc2490e1..9a3e1bebb 100644 --- a/internal/core/ebook_test.go +++ b/internal/core/ebook_test.go @@ -19,7 +19,7 @@ func TestGenerateEbook(t *testing.T) { dstDir := t.TempDir() mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ + Bookmark: model.BookmarkDTO{ ID: 1, Title: "Example Bookmark", HTML: "Example HTML", @@ -39,7 +39,7 @@ func TestGenerateEbook(t *testing.T) { dstDir := t.TempDir() mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ + Bookmark: model.BookmarkDTO{ ID: 1, HasEbook: false, }, @@ -74,7 +74,7 @@ func TestGenerateEbook(t *testing.T) { dstDir := t.TempDir() mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ + Bookmark: model.BookmarkDTO{ ID: 1, HasEbook: false, }, @@ -104,7 +104,7 @@ func TestGenerateEbook(t *testing.T) { t.Run("invalid bookmarkId that return Error", func(t *testing.T) { tempDir := t.TempDir() mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ + Bookmark: model.BookmarkDTO{ ID: 0, HasEbook: false, }, @@ -114,7 +114,7 @@ func TestGenerateEbook(t *testing.T) { bookmark, err := core.GenerateEbook(mockRequest, tempDir) - assert.Equal(t, model.Bookmark{ + assert.Equal(t, model.BookmarkDTO{ ID: 0, HasEbook: false, }, bookmark) @@ -125,7 +125,7 @@ func TestGenerateEbook(t *testing.T) { dstDir := t.TempDir() mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ + Bookmark: model.BookmarkDTO{ ID: 1, HasEbook: false, }, @@ -155,7 +155,7 @@ func TestGenerateEbook(t *testing.T) { tempDir := t.TempDir() mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ + Bookmark: model.BookmarkDTO{ ID: 1, HasEbook: false, }, diff --git a/internal/core/processing.go b/internal/core/processing.go index f30185241..4fb8c6d9c 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -30,7 +30,7 @@ import ( // ProcessRequest is the request for processing bookmark. type ProcessRequest struct { DataDir string - Bookmark model.Bookmark + Bookmark model.BookmarkDTO Content io.Reader ContentType string KeepTitle bool @@ -42,7 +42,7 @@ var ErrNoSupportedImageType = errors.New("unsupported image type") // ProcessBookmark process the bookmark and archive it if needed. // Return three values, is error fatal, and error value. -func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, err error) { +func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr bool, err error) { book = req.Bookmark contentType := req.ContentType diff --git a/internal/core/processing_test.go b/internal/core/processing_test.go index 11e07af3f..a46ca8425 100644 --- a/internal/core/processing_test.go +++ b/internal/core/processing_test.go @@ -104,7 +104,7 @@ func TestDownloadBookImage(t *testing.T) { func TestProcessBookmark(t *testing.T) { t.Run("ProcessRequest with sucssesful result", func(t *testing.T) { t.Run("Normal without image", func(t *testing.T) { - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 1, URL: "https://example.com", Title: "Example", @@ -148,7 +148,7 @@ func TestProcessBookmark(t *testing.T) {

This is an example article

` - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 1, URL: "https://example.com", Title: "Example", @@ -198,7 +198,7 @@ func TestProcessBookmark(t *testing.T) {

This is an example article

` - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 1, URL: "https://example.com", Title: "Example", @@ -231,7 +231,7 @@ func TestProcessBookmark(t *testing.T) { } }) t.Run("ProcessRequest sucssesful with empty title ", func(t *testing.T) { - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 1, URL: "https://example.com", Title: "", @@ -264,7 +264,7 @@ func TestProcessBookmark(t *testing.T) { } }) t.Run("ProcessRequest sucssesful with empty Excerpt", func(t *testing.T) { - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 1, URL: "https://example.com", Title: "", @@ -299,7 +299,7 @@ func TestProcessBookmark(t *testing.T) { t.Run("Specific case", func(t *testing.T) { t.Run("ProcessRequest with ID zero", func(t *testing.T) { - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 0, URL: "https://example.com", Title: "Example", @@ -324,7 +324,7 @@ func TestProcessBookmark(t *testing.T) { t.Run("ProcessRequest that content type not zero", func(t *testing.T) { - bookmark := model.Bookmark{ + bookmark := model.BookmarkDTO{ ID: 1, URL: "https://example.com", Title: "Example", diff --git a/internal/database/database.go b/internal/database/database.go index 02ce72096..d79c56ca0 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -72,10 +72,10 @@ type DB interface { Migrate() error // SaveBookmarks saves bookmarks data to database. - SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) ([]model.Bookmark, error) + SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error) // GetBookmarks fetch list of bookmarks based on submitted options. - GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) + GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) // GetBookmarksCount get count of bookmarks in database. GetBookmarksCount(ctx context.Context, opts GetBookmarksOptions) (int, error) @@ -84,7 +84,7 @@ type DB interface { DeleteBookmarks(ctx context.Context, ids ...int) error // GetBookmark fetches bookmark based on its ID or URL. - GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) + GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) // SaveAccount saves new account in database SaveAccount(ctx context.Context, a model.Account) error diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 101b0d42c..01f8a9d91 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -45,7 +45,7 @@ func testDatabase(t *testing.T, dbFactory testDatabaseFactory) { func testBookmarkAutoIncrement(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -54,7 +54,7 @@ func testBookmarkAutoIncrement(t *testing.T, db DB) { assert.NoError(t, err, "Save bookmarks must not fail") assert.Equal(t, 1, result[0].ID, "Saved bookmark must have ID %d", 1) - book = model.Bookmark{ + book = model.BookmarkDTO{ URL: "https://github.com/go-shiori/obelisk", Title: "obelisk", } @@ -67,7 +67,7 @@ func testBookmarkAutoIncrement(t *testing.T, db DB) { func testCreateBookmark(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/obelisk", Title: "shiori", } @@ -81,7 +81,7 @@ func testCreateBookmark(t *testing.T, db DB) { func testCreateBookmarkWithContent(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/obelisk", Title: "shiori", Content: "Some content", @@ -106,7 +106,7 @@ func testCreateBookmarkWithContent(t *testing.T, db DB) { func testCreateBookmarkWithTag(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/obelisk", Title: "shiori", Tags: []model.Tag{ @@ -126,7 +126,7 @@ func testCreateBookmarkWithTag(t *testing.T, db DB) { func testCreateBookmarkTwice(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -144,7 +144,7 @@ func testCreateBookmarkTwice(t *testing.T, db DB) { func testCreateTwoDifferentBookmarks(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -152,7 +152,7 @@ func testCreateTwoDifferentBookmarks(t *testing.T, db DB) { _, err := db.SaveBookmarks(ctx, true, book) assert.NoError(t, err, "Save first bookmark must not fail") - book = model.Bookmark{ + book = model.BookmarkDTO{ URL: "https://github.com/go-shiori/go-readability", Title: "go-readability", } @@ -163,7 +163,7 @@ func testCreateTwoDifferentBookmarks(t *testing.T, db DB) { func testUpdateBookmark(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -184,7 +184,7 @@ func testUpdateBookmark(t *testing.T, db DB) { func testUpdateBookmarkWithContent(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/obelisk", Title: "shiori", Content: "Some content", @@ -216,7 +216,7 @@ func testUpdateBookmarkWithContent(t *testing.T, db DB) { func testGetBookmark(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -237,13 +237,13 @@ func testGetBookmarkNotExistent(t *testing.T, db DB) { savedBookmark, exists, err := db.GetBookmark(ctx, 1, "") assert.NoError(t, err, "Get bookmark should not fail") assert.False(t, exists, "Bookmark should not exist") - assert.Equal(t, model.Bookmark{}, savedBookmark) + assert.Equal(t, model.BookmarkDTO{}, savedBookmark) } func testGetBookmarks(t *testing.T, db DB) { ctx := context.TODO() - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -266,7 +266,7 @@ func testGetBookmarksWithSQLCharacters(t *testing.T, db DB) { ctx := context.TODO() // _ := 0 - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -296,7 +296,7 @@ func testGetBookmarksCount(t *testing.T, db DB) { ctx := context.TODO() expectedCount := 1 - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } diff --git a/internal/database/mysql.go b/internal/database/mysql.go index 3841a5cbb..ad561e43b 100644 --- a/internal/database/mysql.go +++ b/internal/database/mysql.go @@ -67,8 +67,8 @@ func (db *MySQLDatabase) Migrate() error { // SaveBookmarks saves new or updated bookmarks to database. // Returns the saved ID and error message if any happened. -func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) ([]model.Bookmark, error) { - var result []model.Bookmark +func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error) { + var result []model.BookmarkDTO if err := db.withTx(ctx, func(tx *sqlx.Tx) error { // Prepare statement @@ -218,7 +218,7 @@ func (db *MySQLDatabase) SaveBookmarks(ctx context.Context, create bool, bookmar } // GetBookmarks fetch list of bookmarks based on submitted options. -func (db *MySQLDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) { +func (db *MySQLDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) { // Create initial query columns := []string{ `id`, @@ -330,7 +330,7 @@ func (db *MySQLDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOpti } // Fetch bookmarks - bookmarks := []model.Bookmark{} + bookmarks := []model.BookmarkDTO{} err = db.Select(&bookmarks, query, args...) if err != nil && err != sql.ErrNoRows { return nil, errors.WithStack(err) @@ -502,7 +502,7 @@ func (db *MySQLDatabase) DeleteBookmarks(ctx context.Context, ids ...int) (err e // GetBookmark fetches bookmark based on its ID or URL. // Returns the bookmark and boolean whether it's exist or not. -func (db *MySQLDatabase) GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) { +func (db *MySQLDatabase) GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) { args := []interface{}{id} query := `SELECT id, url, title, excerpt, author, public, @@ -514,7 +514,7 @@ func (db *MySQLDatabase) GetBookmark(ctx context.Context, id int, url string) (m args = append(args, url) } - book := model.Bookmark{} + book := model.BookmarkDTO{} if err := db.GetContext(ctx, &book, query, args...); err != nil && err != sql.ErrNoRows { return book, false, errors.WithStack(err) } diff --git a/internal/database/pg.go b/internal/database/pg.go index 401c4d527..0a36929bf 100644 --- a/internal/database/pg.go +++ b/internal/database/pg.go @@ -68,8 +68,8 @@ func (db *PGDatabase) Migrate() error { // SaveBookmarks saves new or updated bookmarks to database. // Returns the saved ID and error message if any happened. -func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) (result []model.Bookmark, err error) { - result = []model.Bookmark{} +func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) (result []model.BookmarkDTO, err error) { + result = []model.BookmarkDTO{} if err := db.withTx(ctx, func(tx *sqlx.Tx) error { // Prepare statement stmtInsertBook, err := tx.Preparex(`INSERT INTO bookmark @@ -120,7 +120,7 @@ func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks modifiedTime := time.Now().UTC().Format(model.DatabaseDateFormat) // Execute statements - result = []model.Bookmark{} + result = []model.BookmarkDTO{} for _, book := range bookmarks { // URL and title if book.URL == "" { @@ -206,7 +206,7 @@ func (db *PGDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks } // GetBookmarks fetch list of bookmarks based on submitted options. -func (db *PGDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) { +func (db *PGDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) { // Create initial query columns := []string{ `id`, @@ -325,7 +325,7 @@ func (db *PGDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions query = db.Rebind(query) // Fetch bookmarks - bookmarks := []model.Bookmark{} + bookmarks := []model.BookmarkDTO{} err = db.SelectContext(ctx, &bookmarks, query, args...) if err != nil && err != sql.ErrNoRows { return nil, fmt.Errorf("failed to fetch data: %v", err) @@ -511,7 +511,7 @@ func (db *PGDatabase) DeleteBookmarks(ctx context.Context, ids ...int) (err erro // GetBookmark fetches bookmark based on its ID or URL. // Returns the bookmark and boolean whether it's exist or not. -func (db *PGDatabase) GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) { +func (db *PGDatabase) GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) { args := []interface{}{id} query := `SELECT id, url, title, excerpt, author, public, @@ -523,7 +523,7 @@ func (db *PGDatabase) GetBookmark(ctx context.Context, id int, url string) (mode args = append(args, url) } - book := model.Bookmark{} + book := model.BookmarkDTO{} if err := db.GetContext(ctx, &book, query, args...); err != nil && err != sql.ErrNoRows { return book, false, errors.WithStack(err) } diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go index cd6e55bd0..01e649bf6 100644 --- a/internal/database/sqlite.go +++ b/internal/database/sqlite.go @@ -76,8 +76,8 @@ func (db *SQLiteDatabase) Migrate() error { // SaveBookmarks saves new or updated bookmarks to database. // Returns the saved ID and error message if any happened. -func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.Bookmark) ([]model.Bookmark, error) { - var result []model.Bookmark +func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, create bool, bookmarks ...model.BookmarkDTO) ([]model.BookmarkDTO, error) { + var result []model.BookmarkDTO if err := db.withTx(ctx, func(tx *sqlx.Tx) error { // Prepare statement @@ -245,7 +245,7 @@ func (db *SQLiteDatabase) SaveBookmarks(ctx context.Context, create bool, bookma } // GetBookmarks fetch list of bookmarks based on submitted options. -func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.Bookmark, error) { +func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOptions) ([]model.BookmarkDTO, error) { // Create initial query query := `SELECT b.id, @@ -362,7 +362,7 @@ func (db *SQLiteDatabase) GetBookmarks(ctx context.Context, opts GetBookmarksOpt } // Fetch bookmarks - bookmarks := []model.Bookmark{} + bookmarks := []model.BookmarkDTO{} err = db.SelectContext(ctx, &bookmarks, query, args...) if err != nil && err != sql.ErrNoRows { return nil, errors.WithStack(err) @@ -622,7 +622,7 @@ func (db *SQLiteDatabase) DeleteBookmarks(ctx context.Context, ids ...int) error // GetBookmark fetches bookmark based on its ID or URL. // Returns the bookmark and boolean whether it's exist or not. -func (db *SQLiteDatabase) GetBookmark(ctx context.Context, id int, url string) (model.Bookmark, bool, error) { +func (db *SQLiteDatabase) GetBookmark(ctx context.Context, id int, url string) (model.BookmarkDTO, bool, error) { args := []interface{}{id} query := `SELECT b.id, b.url, b.title, b.excerpt, b.author, b.public, b.modified, @@ -636,7 +636,7 @@ func (db *SQLiteDatabase) GetBookmark(ctx context.Context, id int, url string) ( args = append(args, url) } - book := model.Bookmark{} + book := model.BookmarkDTO{} if err := db.GetContext(ctx, &book, query, args...); err != nil && err != sql.ErrNoRows { return book, false, errors.WithStack(err) } diff --git a/internal/database/sqlite_test.go b/internal/database/sqlite_test.go index 7576058c5..509df0e5c 100644 --- a/internal/database/sqlite_test.go +++ b/internal/database/sqlite_test.go @@ -50,7 +50,7 @@ func testSqliteGetBookmarksWithDash(t *testing.T) { db, err := sqliteTestDatabaseFactory(ctx) assert.NoError(t, err) - book := model.Bookmark{ + book := model.BookmarkDTO{ URL: "https://github.com/go-shiori/shiori", Title: "shiori", } @@ -58,7 +58,7 @@ func testSqliteGetBookmarksWithDash(t *testing.T) { _, err = db.SaveBookmarks(ctx, true, book) assert.NoError(t, err, "Save bookmarks must not fail") - book = model.Bookmark{ + book = model.BookmarkDTO{ URL: "https://github.com/jamiehannaford/what-happens-when-k8s", Title: "what-happens-when-k8s", } diff --git a/internal/config/dependencies.go b/internal/dependencies/dependencies.go similarity index 52% rename from internal/config/dependencies.go rename to internal/dependencies/dependencies.go index ed0b7afdf..aa824b410 100644 --- a/internal/config/dependencies.go +++ b/internal/dependencies/dependencies.go @@ -1,6 +1,7 @@ -package config +package dependencies import ( + "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/database" "github.com/go-shiori/shiori/internal/domains" "github.com/sirupsen/logrus" @@ -9,14 +10,15 @@ import ( type Dependencies struct { Log *logrus.Logger Database database.DB - Config *Config + Config *config.Config Domains struct { - Auth domains.AccountsDomain - Archiver domains.ArchiverDomain + Auth domains.AccountsDomain + Archiver domains.ArchiverDomain + Bookmarks domains.BookmarksDomain } } -func NewDependencies(log *logrus.Logger, db database.DB, cfg *Config) *Dependencies { +func NewDependencies(log *logrus.Logger, db database.DB, cfg *config.Config) *Dependencies { return &Dependencies{ Log: log, Config: cfg, diff --git a/internal/domains/archiver.go b/internal/domains/archiver.go index 8b44a5b9d..706eedf8c 100644 --- a/internal/domains/archiver.go +++ b/internal/domains/archiver.go @@ -2,12 +2,12 @@ package domains import ( "fmt" - "os" "path/filepath" "strconv" "github.com/go-shiori/shiori/internal/core" "github.com/go-shiori/shiori/internal/model" + "github.com/go-shiori/warc" "github.com/sirupsen/logrus" ) @@ -16,7 +16,7 @@ type ArchiverDomain struct { logger *logrus.Logger } -func (d *ArchiverDomain) DownloadBookmarkArchive(book model.Bookmark) (*model.Bookmark, error) { +func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model.BookmarkDTO, error) { content, contentType, err := core.DownloadBookmark(book.URL) if err != nil { return nil, fmt.Errorf("error downloading url: %s", err) @@ -39,15 +39,14 @@ func (d *ArchiverDomain) DownloadBookmarkArchive(book model.Bookmark) (*model.Bo return &result, nil } -func (d *ArchiverDomain) GetBookmarkArchive(book model.Bookmark) error { +func (d *ArchiverDomain) GetBookmarkArchive(book *model.BookmarkDTO) (*warc.Archive, error) { archivePath := filepath.Join(d.dataDir, "archive", strconv.Itoa(book.ID)) - info, err := os.Stat(archivePath) - if !os.IsNotExist(err) && !info.IsDir() { - return fmt.Errorf("archive not found") + if !FileExists(archivePath) { + return nil, fmt.Errorf("archive not found") } - return nil + return warc.Open(archivePath) } func NewArchiverDomain(logger *logrus.Logger, dataDir string) ArchiverDomain { diff --git a/internal/domains/bookmarks.go b/internal/domains/bookmarks.go new file mode 100644 index 000000000..4d17fdf3e --- /dev/null +++ b/internal/domains/bookmarks.go @@ -0,0 +1,133 @@ +package domains + +import ( + "context" + "fmt" + "html/template" + "net/url" + "path" + "path/filepath" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/model" + "github.com/go-shiori/warc" + "github.com/sirupsen/logrus" +) + +type BookmarksDomain struct { + logger *logrus.Logger + db database.DB + cfg *config.Config +} + +func (d *BookmarksDomain) HasEbook(b *model.BookmarkDTO) bool { + ebookPath := filepath.Join(d.cfg.Storage.DataDir, "ebook", strconv.Itoa(b.ID)+".epub") + return FileExists(ebookPath) +} + +func (d *BookmarksDomain) HasArchive(b *model.BookmarkDTO) bool { + archivePath := filepath.Join(d.cfg.Storage.DataDir, "archive", strconv.Itoa(b.ID)) + return FileExists(archivePath) +} + +func (d *BookmarksDomain) GetThumbnailPath(b *model.BookmarkDTO) string { + return filepath.Join(d.cfg.Storage.DataDir, "thumb", strconv.Itoa(b.ID)) +} + +func (d *BookmarksDomain) HasThumbnail(b *model.BookmarkDTO) bool { + return FileExists(d.GetThumbnailPath(b)) +} + +func (d *BookmarksDomain) GetBookmark(ctx context.Context, id model.DBID) (*model.BookmarkDTO, error) { + bookmark, _, err := d.db.GetBookmark(ctx, int(id), "") + if err != nil { + return nil, fmt.Errorf("failed to get bookmark: %w", err) + } + + // Check if it has ebook and archive. + bookmark.HasEbook = d.HasEbook(&bookmark) + bookmark.HasArchive = d.HasArchive(&bookmark) + + return &bookmark, nil +} + +// GetBookmarkContentsFromArchive gets the HTML contents of a bookmark linking assets to the ones +// archived if the bookmark has an archived version. +func (d *BookmarksDomain) GetBookmarkContentsFromArchive(bookmark *model.BookmarkDTO) (template.HTML, error) { + if !bookmark.HasArchive { + return template.HTML(bookmark.HTML), nil + } + + // Open archive, look in cache first + archivePath := filepath.Join(d.cfg.Storage.DataDir, "archive", fmt.Sprintf("%d", bookmark.ID)) + archive, err := warc.Open(archivePath) + if err != nil { + return "", fmt.Errorf("failed to open archive: %w", err) + } + + // Find all image and convert its source to use the archive URL. + createArchivalURL := func(archivalName string) string { + var archivalURL url.URL + archivalURL.Path = path.Join(d.cfg.Http.RootPath, "bookmark", fmt.Sprintf("%d", bookmark.ID), "archive", archivalName) + return archivalURL.String() + } + + buffer := strings.NewReader(bookmark.HTML) + doc, err := goquery.NewDocumentFromReader(buffer) + if err != nil { + return "", fmt.Errorf("failed to parse HTML: %w", err) + } + + doc.Find("img, picture, figure, source").Each(func(_ int, node *goquery.Selection) { + // Get the needed attributes + src, _ := node.Attr("src") + strSrcSets, _ := node.Attr("srcset") + + // Convert `src` attributes + if src != "" { + archivalName := getArchiveFileBasename(src) + if archivalName != "" && archive.HasResource(archivalName) { + node.SetAttr("src", createArchivalURL(archivalName)) + } + } + + // Split srcset by comma, then process it like any URLs + srcSets := strings.Split(strSrcSets, ",") + for i, srcSet := range srcSets { + srcSet = strings.TrimSpace(srcSet) + parts := strings.SplitN(srcSet, " ", 2) + if parts[0] == "" { + continue + } + + archivalName := getArchiveFileBasename(parts[0]) + if archivalName != "" && archive.HasResource(archivalName) { + archivalURL := createArchivalURL(archivalName) + srcSets[i] = strings.Replace(srcSets[i], parts[0], archivalURL, 1) + } + } + + if len(srcSets) > 0 { + node.SetAttr("srcset", strings.Join(srcSets, ",")) + } + }) + + bookmark.HTML, err = goquery.OuterHtml(doc.Selection) + if err != nil { + return template.HTML(bookmark.HTML), fmt.Errorf("failed to get HTML: %w", err) + } + + return template.HTML(bookmark.HTML), nil +} + +func NewBookmarksDomain(logger *logrus.Logger, db database.DB, cfg *config.Config) BookmarksDomain { + return BookmarksDomain{ + logger: logger, + db: db, + cfg: cfg, + } +} diff --git a/internal/domains/filesystem.go b/internal/domains/filesystem.go new file mode 100644 index 000000000..e28ec8da2 --- /dev/null +++ b/internal/domains/filesystem.go @@ -0,0 +1,24 @@ +package domains + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +type FilesystemDomain struct { + logger *logrus.Logger + dataDir string +} + +func (d *FilesystemDomain) FileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +func NewFilesystemDomain(logger *logrus.Logger, dataDir string) FilesystemDomain { + return FilesystemDomain{ + logger: logger, + dataDir: dataDir, + } +} diff --git a/internal/domains/utils.go b/internal/domains/utils.go new file mode 100644 index 000000000..1b45a85d8 --- /dev/null +++ b/internal/domains/utils.go @@ -0,0 +1,46 @@ +package domains + +import ( + "net/url" + "os" + "regexp" + "strings" +) + +var rxRepeatedStrip = regexp.MustCompile(`(?i)-+`) + +func FileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +// getArchiveFileBasename converts an URL into an archival name. +func getArchiveFileBasename(src string) string { + archivalURL := src + + // Some URL have its query or path escaped, e.g. Wikipedia and Dev.to. + // For example, Wikipedia's stylesheet looks like this : + // load.php?lang=en&modules=ext.3d.styles%7Cext.cite.styles%7Cext.uls.interlanguage + // However, when browser download it, it will be registered as unescaped query : + // load.php?lang=en&modules=ext.3d.styles|ext.cite.styles|ext.uls.interlanguage + // So, for archival URL, we need to unescape the query and path first. + tmp, err := url.Parse(src) + if err == nil { + unescapedQuery, _ := url.QueryUnescape(tmp.RawQuery) + if unescapedQuery != "" { + tmp.RawQuery = unescapedQuery + } + + archivalURL = tmp.String() + archivalURL = strings.Replace(archivalURL, tmp.EscapedPath(), tmp.Path, 1) + } + + archivalURL = strings.ReplaceAll(archivalURL, "://", "/") + archivalURL = strings.ReplaceAll(archivalURL, "?", "-") + archivalURL = strings.ReplaceAll(archivalURL, "#", "-") + archivalURL = strings.ReplaceAll(archivalURL, "/", "-") + archivalURL = strings.ReplaceAll(archivalURL, " ", "-") + archivalURL = rxRepeatedStrip.ReplaceAllString(archivalURL, "-") + + return archivalURL +} diff --git a/internal/http/middleware/auth.go b/internal/http/middleware/auth.go index 467536e33..cb3da436f 100644 --- a/internal/http/middleware/auth.go +++ b/internal/http/middleware/auth.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/gin-gonic/gin" - "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/context" "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" @@ -14,7 +14,7 @@ import ( // AuthMiddleware provides basic authentication capabilities to all routes underneath // its usage, only allowing authenticated users access and set a custom local context // `account` with the account model for the logged in user. -func AuthMiddleware(deps *config.Dependencies) gin.HandlerFunc { +func AuthMiddleware(deps *dependencies.Dependencies) gin.HandlerFunc { return func(c *gin.Context) { token := getTokenFromHeader(c) if token == "" { diff --git a/internal/http/response/file.go b/internal/http/response/file.go new file mode 100644 index 000000000..d97aa03a8 --- /dev/null +++ b/internal/http/response/file.go @@ -0,0 +1,23 @@ +package response + +import ( + "fmt" + "net/http" + "os" + + "github.com/gin-gonic/gin" +) + +// SendFile sends file to client with caching header +func SendFile(c *gin.Context, path string) { + c.Header("Cache-Control", "public, max-age=86400") + + info, err := os.Stat(path) + if err != nil { + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + c.Header("ETag", fmt.Sprintf("W/%x-%x", info.ModTime().Unix(), info.Size())) + c.File(path) +} diff --git a/internal/http/routes/api/v1/api.go b/internal/http/routes/api/v1/api.go index 65f8bd563..dbbea86aa 100644 --- a/internal/http/routes/api/v1/api.go +++ b/internal/http/routes/api/v1/api.go @@ -2,7 +2,7 @@ package api_v1 import ( "github.com/gin-gonic/gin" - "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/middleware" "github.com/go-shiori/shiori/internal/model" "github.com/sirupsen/logrus" @@ -10,7 +10,7 @@ import ( type APIRoutes struct { logger *logrus.Logger - deps *config.Dependencies + deps *dependencies.Dependencies loginHandler model.LegacyLoginHandler } @@ -31,7 +31,7 @@ func (s *APIRoutes) handle(g *gin.RouterGroup, path string, routes model.Routes) routes.Setup(group) } -func NewAPIRoutes(logger *logrus.Logger, deps *config.Dependencies, loginHandler model.LegacyLoginHandler) *APIRoutes { +func NewAPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies, loginHandler model.LegacyLoginHandler) *APIRoutes { return &APIRoutes{ logger: logger, deps: deps, diff --git a/internal/http/routes/api/v1/auth.go b/internal/http/routes/api/v1/auth.go index 0c57008da..10f8ad3dc 100644 --- a/internal/http/routes/api/v1/auth.go +++ b/internal/http/routes/api/v1/auth.go @@ -6,7 +6,7 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/context" "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" @@ -15,7 +15,7 @@ import ( type AuthAPIRoutes struct { logger *logrus.Logger - deps *config.Dependencies + deps *dependencies.Dependencies legacyLoginHandler model.LegacyLoginHandler } @@ -189,7 +189,7 @@ func (r *AuthAPIRoutes) settingsHandler(c *gin.Context) { response.Send(c, http.StatusOK, ctx.GetAccount()) } -func NewAuthAPIRoutes(logger *logrus.Logger, deps *config.Dependencies, loginHandler model.LegacyLoginHandler) *AuthAPIRoutes { +func NewAuthAPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies, loginHandler model.LegacyLoginHandler) *AuthAPIRoutes { return &AuthAPIRoutes{ logger: logger, deps: deps, diff --git a/internal/http/routes/api/v1/bookmarks.go b/internal/http/routes/api/v1/bookmarks.go index 7b7da45d7..747ff3956 100644 --- a/internal/http/routes/api/v1/bookmarks.go +++ b/internal/http/routes/api/v1/bookmarks.go @@ -13,6 +13,7 @@ import ( "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/core" "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/context" "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" @@ -21,7 +22,7 @@ import ( type BookmarksAPIRoutes struct { logger *logrus.Logger - deps *config.Dependencies + deps *dependencies.Dependencies } func (r *BookmarksAPIRoutes) Setup(g *gin.RouterGroup) model.Routes { @@ -73,8 +74,8 @@ type apiCreateBookmarkPayload struct { Async bool `json:"async"` } -func (payload *apiCreateBookmarkPayload) ToBookmark() (*model.Bookmark, error) { - bookmark := &model.Bookmark{ +func (payload *apiCreateBookmarkPayload) ToBookmark() (*model.BookmarkDTO, error) { + bookmark := &model.BookmarkDTO{ URL: payload.URL, Title: payload.Title, Excerpt: payload.Excerpt, @@ -188,7 +189,7 @@ func (r *BookmarksAPIRoutes) deleteHandler(c *gin.Context) { response.Send(c, 200, "Bookmark deleted") } -func NewBookmarksPIRoutes(logger *logrus.Logger, deps *config.Dependencies) *BookmarksAPIRoutes { +func NewBookmarksPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarksAPIRoutes { return &BookmarksAPIRoutes{ logger: logger, deps: deps, @@ -261,7 +262,7 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) { book.CreateArchive = payload.CreateArchive book.CreateEbook = payload.CreateEbook - go func(i int, book model.Bookmark, keep_metadata bool) { + go func(i int, book model.BookmarkDTO, keep_metadata bool) { // Make sure to finish the WG defer wg.Done() diff --git a/internal/http/routes/api/v1/tags.go b/internal/http/routes/api/v1/tags.go index a154d76d1..bbb253be9 100644 --- a/internal/http/routes/api/v1/tags.go +++ b/internal/http/routes/api/v1/tags.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" "github.com/sirupsen/logrus" @@ -12,7 +12,7 @@ import ( type TagsAPIRoutes struct { logger *logrus.Logger - deps *config.Dependencies + deps *dependencies.Dependencies } func (r *TagsAPIRoutes) Setup(g *gin.RouterGroup) model.Routes { @@ -62,7 +62,7 @@ func (r *TagsAPIRoutes) createHandler(c *gin.Context) { response.Send(c, http.StatusCreated, nil) } -func NewTagsPIRoutes(logger *logrus.Logger, deps *config.Dependencies) *TagsAPIRoutes { +func NewTagsPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *TagsAPIRoutes { return &TagsAPIRoutes{ logger: logger, deps: deps, diff --git a/internal/http/routes/bookmark.go b/internal/http/routes/bookmark.go index 6db6a5ba2..397a6a60b 100644 --- a/internal/http/routes/bookmark.go +++ b/internal/http/routes/bookmark.go @@ -1,13 +1,19 @@ package routes import ( + "bytes" + "compress/gzip" + "html/template" "net/http" "strconv" + "strings" fp "path/filepath" + "github.com/PuerkitoBio/goquery" "github.com/gin-gonic/gin" "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/context" "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" @@ -17,54 +23,152 @@ import ( type BookmarkRoutes struct { logger *logrus.Logger - deps *config.Dependencies + deps *dependencies.Dependencies } func (r *BookmarkRoutes) Setup(group *gin.RouterGroup) model.Routes { - //group.GET("/:id/archive", r.bookmarkArchiveHandler) - //group.GET("/:id/content", r.bookmarkContentHandler) + group.GET("/:id/archive", r.bookmarkArchiveHandler) + group.GET("/:id/archive/*filepath", r.bookmarkArchiveHandler) + group.GET("/:id/content", r.bookmarkContentHandler) + group.GET("/:id/thumb", r.bookmarkThumbnailHandler) group.GET("/:id/ebook", r.bookmarkEbookHandler) + return r } -// func (r *BookmarkRoutes) bookmarkContentHandler(c *gin.Context) { -// ctx := context.NewContextFromGin(c) +func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, error) { + bookmarkIDParam, present := c.Params.Get("id") + if !present { + response.SendError(c.Context, 400, "Invalid bookmark ID") + } + + bookmarkID, err := strconv.Atoi(bookmarkIDParam) + if err != nil { + r.logger.WithError(err).Error("error parsing bookmark ID parameter") + response.SendInternalServerError(c.Context) + return nil, err + } + + if bookmarkID == 0 { + response.SendError(c.Context, 404, nil) + return nil, err + } + + bookmark, err := r.deps.Domains.Bookmarks.GetBookmark(c.Context, model.DBID(bookmarkID)) + if err != nil { + response.SendError(c.Context, 404, nil) + return nil, err + } + + if bookmark.Public != 1 && !c.UserIsLogged() { + response.SendError(c.Context, http.StatusForbidden, nil) + return nil, err + } + + return bookmark, nil +} + +func (r *BookmarkRoutes) bookmarkContentHandler(c *gin.Context) { + ctx := context.NewContextFromGin(c) + + bookmark, err := r.getBookmark(ctx) + if err != nil { + return + } + + ctx.HTML(http.StatusOK, "content.html", gin.H{ + "RootPath": r.deps.Config.Http.RootPath, + "Version": model.BuildVersion, + "Book": bookmark, + "HTML": template.HTML(bookmark.HTML), + }) +} + +func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) { + ctx := context.NewContextFromGin(c) + + bookmark, err := r.getBookmark(ctx) + if err != nil { + return + } -// bookmarkIDParam, present := c.Params.Get("id") -// if !present { -// response.SendError(c, 400, "Invalid bookmark ID") -// return -// } + if !r.deps.Domains.Bookmarks.HasArchive(bookmark) { + response.SendError(c, http.StatusNotFound, nil) + return + } -// bookmarkID, err := strconv.Atoi(bookmarkIDParam) -// if err != nil { -// r.logger.WithError(err).Error("error parsing bookmark ID parameter") -// response.SendInternalServerError(c) -// return -// } + resourcePath, _ := c.Params.Get("filepath") + resourcePath = strings.TrimPrefix(resourcePath, "/") -// if bookmarkID == 0 { -// response.SendError(c, 404, nil) -// return -// } + archive, err := r.deps.Domains.Archiver.GetBookmarkArchive(bookmark) + if err != nil { + r.logger.WithError(err).Error("error opening archive") + response.SendInternalServerError(c) + return + } -// bookmark, found, err := r.deps.Database.GetBookmark(c, bookmarkID, "") -// if err != nil || !found { -// response.SendError(c, 404, nil) -// return -// } + content, resourceContentType, err := archive.Read(resourcePath) + if err != nil { + r.logger.WithError(err).Error("error reading archive file") + response.SendInternalServerError(c) + return + } -// if bookmark.Public != 1 && !ctx.UserIsLogged() { -// response.SendError(c, http.StatusForbidden, nil) -// return -// } + // If this is HTML and root, inject shiori header + if strings.Contains(strings.ToLower(resourceContentType), "text/html") && resourcePath == "" { + // Extract gzip + buffer := bytes.NewBuffer(content) + gzipReader, err := gzip.NewReader(buffer) + if err != nil { + r.logger.WithError(err).Error("error creating gzip reader") + response.SendInternalServerError(c) + return + } + + // Parse gzipped content + doc, err := goquery.NewDocumentFromReader(gzipReader) + if err != nil { + r.logger.WithError(err).Error("error parsing gzipped content") + response.SendInternalServerError(c) + return + } + + // Revert back to HTML + outerHTML, err := goquery.OuterHtml(doc.Selection) + if err != nil { + r.logger.WithError(err).Error("error creating outer HTML") + response.SendInternalServerError(c) + return + } + + // Gzip it again and send to response writer + c.Data(http.StatusOK, "text/html", []byte(outerHTML)) + return + } + + // Serve content + c.Header("Content-Encoding", "gzip") + // TODO: Set ETag header + c.Data(http.StatusOK, resourceContentType, content) +} + +func (r *BookmarkRoutes) bookmarkThumbnailHandler(c *gin.Context) { + ctx := context.NewContextFromGin(c) + + bookmark, err := r.getBookmark(ctx) + if err != nil { + return + } -// response.Send(c, 200, bookmark.Content) -// } + if !r.deps.Domains.Bookmarks.HasThumbnail(bookmark) { + response.SendError(c, http.StatusNotFound, nil) + return + } -// func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) {} + response.SendFile(c, r.deps.Domains.Bookmarks.GetThumbnailPath(bookmark)) +} -func NewBookmarkRoutes(logger *logrus.Logger, deps *config.Dependencies) *BookmarkRoutes { +func NewBookmarkRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarkRoutes { return &BookmarkRoutes{ logger: logger, deps: deps, diff --git a/internal/http/routes/frontend.go b/internal/http/routes/frontend.go index d1ad894cc..7504dc871 100644 --- a/internal/http/routes/frontend.go +++ b/internal/http/routes/frontend.go @@ -2,11 +2,9 @@ package routes import ( "embed" - "html/template" "net/http" "path/filepath" - "github.com/gin-contrib/gzip" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" "github.com/go-shiori/shiori/internal/config" @@ -48,21 +46,8 @@ type FrontendRoutes struct { cfg *config.Config } -func (r *FrontendRoutes) loadTemplates(e *gin.Engine) { - tmpl, err := template.New("html").Delims("$$", "$$").ParseFS(views.Templates, "*.html") - if err != nil { - r.logger.WithError(err).Error("Failed to parse templates") - return - } - e.SetHTMLTemplate(tmpl) -} - func (r *FrontendRoutes) Setup(e *gin.Engine) { group := e.Group("/") - e.Delims("$$", "$$") - r.loadTemplates(e) - // e.LoadHTMLGlob("internal/view/*.html") - group.Use(gzip.Gzip(gzip.DefaultCompression)) group.GET("/login", func(ctx *gin.Context) { ctx.HTML(http.StatusOK, "login.html", gin.H{ "RootPath": r.cfg.Http.RootPath, diff --git a/internal/http/routes/frontend_test.go b/internal/http/routes/frontend_test.go index 3ae2474ae..8ba4f1ad7 100644 --- a/internal/http/routes/frontend_test.go +++ b/internal/http/routes/frontend_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-shiori/shiori/internal/http/templates" "github.com/go-shiori/shiori/internal/testutil" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" @@ -18,6 +19,8 @@ func TestFrontendRoutes(t *testing.T) { cfg, _ := testutil.GetTestConfigurationAndDependencies(t, context.Background(), logger) g := gin.Default() + templates.SetupTemplates(g) + router := NewFrontendRoutes(logger, cfg) router.Setup(g) diff --git a/internal/http/routes/legacy.go b/internal/http/routes/legacy.go index 134742ef9..899feaeaf 100644 --- a/internal/http/routes/legacy.go +++ b/internal/http/routes/legacy.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/go-shiori/shiori/internal/webserver" "github.com/gofrs/uuid/v5" @@ -17,7 +18,7 @@ import ( type LegacyAPIRoutes struct { logger *logrus.Logger cfg *config.Config - deps *config.Dependencies + deps *dependencies.Dependencies legacyHandler *webserver.Handler } @@ -80,12 +81,12 @@ func (r *LegacyAPIRoutes) Setup(g *gin.Engine) { legacyGroup.POST("/api/logout", r.handle(r.legacyHandler.ApiLogout)) // router.GET(jp("/bookmark/:id/thumb"), withLogging(hdl.serveThumbnailImage)) - legacyGroup.GET("/bookmark/:id/thumb", r.handle(r.legacyHandler.ServeThumbnailImage)) + // legacyGroup.GET("/bookmark/:id/thumb", r.handle(r.legacyHandler.ServeThumbnailImage)) // router.GET(jp("/bookmark/:id/content"), withLogging(hdl.serveBookmarkContent)) - legacyGroup.GET("/bookmark/:id/content", r.handle(r.legacyHandler.ServeBookmarkContent)) + // legacyGroup.GET("/bookmark/:id/content", r.handle(r.legacyHandler.ServeBookmarkContent)) // router.GET(jp("/bookmark/:id/archive/*filepath"), withLogging(hdl.serveBookmarkArchive)) - legacyGroup.GET("/bookmark/:id/archive/*filepath", r.handle(r.legacyHandler.ServeBookmarkArchive)) - // legacyGroup.GET("/bookmark/:id/archive/", r.handle(r.legacyHandler.ServeBookmarkArchive)) + // legacyGroup.GET("/legacy/:id/archive/", r.handle(r.legacyHandler.ServeBookmarkArchive)) + // legacyGroup.GET("/legacy/:id/archive/*filepath", r.handle(r.legacyHandler.ServeBookmarkArchive)) // router.GET(jp("/api/tags"), withLogging(hdl.apiGetTags)) legacyGroup.GET("/api/tags", r.handle(r.legacyHandler.ApiGetTags)) @@ -116,7 +117,7 @@ func (r *LegacyAPIRoutes) Setup(g *gin.Engine) { legacyGroup.DELETE("/api/accounts", r.handle(r.legacyHandler.ApiDeleteAccount)) } -func NewLegacyAPIRoutes(logger *logrus.Logger, deps *config.Dependencies, cfg *config.Config) *LegacyAPIRoutes { +func NewLegacyAPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies, cfg *config.Config) *LegacyAPIRoutes { return &LegacyAPIRoutes{ logger: logger, cfg: cfg, diff --git a/internal/http/server.go b/internal/http/server.go index e0dff1597..cc3702464 100644 --- a/internal/http/server.go +++ b/internal/http/server.go @@ -8,12 +8,15 @@ import ( "os/signal" "syscall" + "github.com/gin-contrib/gzip" "github.com/gin-contrib/requestid" "github.com/gin-gonic/gin" "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/middleware" "github.com/go-shiori/shiori/internal/http/routes" api_v1 "github.com/go-shiori/shiori/internal/http/routes/api/v1" + "github.com/go-shiori/shiori/internal/http/templates" "github.com/go-shiori/shiori/internal/model" "github.com/sirupsen/logrus" ginlogrus "github.com/toorop/gin-logrus" @@ -25,13 +28,17 @@ type HttpServer struct { logger *logrus.Logger } -func (s *HttpServer) Setup(cfg *config.Config, deps *config.Dependencies) *HttpServer { +func (s *HttpServer) Setup(cfg *config.Config, deps *dependencies.Dependencies) (*HttpServer, error) { if !cfg.Development { gin.SetMode(gin.ReleaseMode) } s.engine = gin.New() + templates.SetupTemplates(s.engine) + + s.engine.Use(gzip.Gzip(gzip.DefaultCompression)) + s.engine.Use(requestid.New()) if cfg.Http.AccessLog { @@ -60,7 +67,7 @@ func (s *HttpServer) Setup(cfg *config.Config, deps *config.Dependencies) *HttpS s.http.Handler = s.engine s.http.Addr = fmt.Sprintf("%s%d", cfg.Http.Address, cfg.Http.Port) - return s + return s, nil } func (s *HttpServer) handle(path string, routes model.Routes) { diff --git a/internal/http/templates/templates.go b/internal/http/templates/templates.go new file mode 100644 index 000000000..9e0f89e9a --- /dev/null +++ b/internal/http/templates/templates.go @@ -0,0 +1,25 @@ +package templates + +import ( + "fmt" + "html/template" + + "github.com/gin-gonic/gin" + views "github.com/go-shiori/shiori/internal/view" +) + +const ( + leftTemplateDelim = "$$" + rightTemplateDelim = "$$" +) + +// SetupTemplates sets up the templates for the webserver. +func SetupTemplates(engine *gin.Engine) error { + engine.Delims(leftTemplateDelim, rightTemplateDelim) + tmpl, err := template.New("html").Delims(leftTemplateDelim, rightTemplateDelim).ParseFS(views.Templates, "*.html") + if err != nil { + return fmt.Errorf("failed to parse templates: %w", err) + } + engine.SetHTMLTemplate(tmpl) + return nil +} diff --git a/internal/model/account.go b/internal/model/account.go index 5004401d7..3128041f1 100644 --- a/internal/model/account.go +++ b/internal/model/account.go @@ -27,6 +27,23 @@ type UserConfig struct { MakePublic bool `json:"MakePublic"` } +func (c *UserConfig) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + json.Unmarshal(v, &c) + return nil + case string: + json.Unmarshal([]byte(v), &c) + return nil + default: + return fmt.Errorf("unsupported type: %T", v) + } +} + +func (c UserConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + // ToDTO converts Account to AccountDTO. func (a Account) ToDTO() AccountDTO { return AccountDTO{ @@ -44,20 +61,3 @@ type AccountDTO struct { Owner bool `json:"owner"` Config UserConfig `json:"config"` } - -func (c *UserConfig) Scan(value interface{}) error { - switch v := value.(type) { - case []byte: - json.Unmarshal(v, &c) - return nil - case string: - json.Unmarshal([]byte(v), &c) - return nil - default: - return fmt.Errorf("unsupported type: %T", v) - } -} - -func (c UserConfig) Value() (driver.Value, error) { - return json.Marshal(c) -} diff --git a/internal/model/model.go b/internal/model/bookmark.go similarity index 73% rename from internal/model/model.go rename to internal/model/bookmark.go index 9ed0d4cac..e2dcd5c44 100644 --- a/internal/model/model.go +++ b/internal/model/bookmark.go @@ -1,15 +1,8 @@ package model -// Tag is the tag for a bookmark. -type Tag struct { - ID int `db:"id" json:"id"` - Name string `db:"name" json:"name"` - NBookmarks int `db:"n_bookmarks" json:"nBookmarks,omitempty"` - Deleted bool `json:"-"` -} - -// Bookmark is the record for an URL. -type Bookmark struct { +// BookmarkDTO is the bookmark object representation in database and the data transfer object +// at the same time, pending a refactor to two separate object to represent each role. +type BookmarkDTO struct { ID int `db:"id" json:"id"` URL string `db:"url" json:"url"` Title string `db:"title" json:"title"` @@ -21,9 +14,9 @@ type Bookmark struct { HTML string `db:"html" json:"html,omitempty"` ImageURL string `db:"image_url" json:"imageURL"` HasContent bool `db:"has_content" json:"hasContent"` + Tags []Tag `json:"tags"` HasArchive bool `json:"hasArchive"` HasEbook bool `json:"hasEbook"` - Tags []Tag `json:"tags"` CreateArchive bool `json:"create_archive"` CreateEbook bool `json:"create_ebook"` } diff --git a/internal/model/db.go b/internal/model/db.go new file mode 100644 index 000000000..62d78dd95 --- /dev/null +++ b/internal/model/db.go @@ -0,0 +1,3 @@ +package model + +type DBID int diff --git a/internal/model/tag.go b/internal/model/tag.go new file mode 100644 index 000000000..84181408f --- /dev/null +++ b/internal/model/tag.go @@ -0,0 +1,9 @@ +package model + +// Tag is the tag for a bookmark. +type Tag struct { + ID int `db:"id" json:"id"` + Name string `db:"name" json:"name"` + NBookmarks int `db:"n_bookmarks" json:"nBookmarks,omitempty"` + Deleted bool `json:"-"` +} diff --git a/internal/testutil/shiori.go b/internal/testutil/shiori.go index eb97e2850..479e5f914 100644 --- a/internal/testutil/shiori.go +++ b/internal/testutil/shiori.go @@ -7,12 +7,13 @@ import ( "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/domains" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) -func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logger *logrus.Logger) (*config.Config, *config.Dependencies) { +func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logger *logrus.Logger) (*config.Config, *dependencies.Dependencies) { tmp, err := os.CreateTemp("", "") require.NoError(t, err) @@ -28,7 +29,7 @@ func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logg cfg.Storage.DataDir = tempDir - deps := config.NewDependencies(logger, db, cfg) + deps := dependencies.NewDependencies(logger, db, cfg) deps.Database = db deps.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db) deps.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir) diff --git a/internal/view/archive.html b/internal/view/archive.html new file mode 100644 index 000000000..bd1e1b532 --- /dev/null +++ b/internal/view/archive.html @@ -0,0 +1,20 @@ + + + + + + $$.Book.Title$$ + + + +
+ +
+ View Original + $$if .HasContent$$ + View Readable + $$end$$ +
+ + + diff --git a/internal/view/content.html b/internal/view/content.html index a966a134a..7f9899e6a 100644 --- a/internal/view/content.html +++ b/internal/view/content.html @@ -36,7 +36,7 @@
- $$html .Book.HTML$$ + $$.HTML$$
diff --git a/internal/webserver/handler-api-ext.go b/internal/webserver/handler-api-ext.go index ea75aee13..1a936b5d9 100644 --- a/internal/webserver/handler-api-ext.go +++ b/internal/webserver/handler-api-ext.go @@ -25,7 +25,7 @@ func (h *Handler) ApiInsertViaExtension(w http.ResponseWriter, r *http.Request, checkError(err) // Decode request - request := model.Bookmark{} + request := model.BookmarkDTO{} err = json.NewDecoder(r.Body).Decode(&request) checkError(err) @@ -126,7 +126,7 @@ func (h *Handler) ApiDeleteViaExtension(w http.ResponseWriter, r *http.Request, checkError(err) // Decode request - request := model.Bookmark{} + request := model.BookmarkDTO{} err = json.NewDecoder(r.Body).Decode(&request) checkError(err) diff --git a/internal/webserver/handler-api.go b/internal/webserver/handler-api.go index 04e7d8b9a..85a83e092 100644 --- a/internal/webserver/handler-api.go +++ b/internal/webserver/handler-api.go @@ -20,7 +20,7 @@ import ( "golang.org/x/crypto/bcrypt" ) -func downloadBookmarkContent(book *model.Bookmark, dataDir string, request *http.Request, keepTitle, keepExcerpt bool) (*model.Bookmark, error) { +func downloadBookmarkContent(book *model.BookmarkDTO, dataDir string, request *http.Request, keepTitle, keepExcerpt bool) (*model.BookmarkDTO, error) { content, contentType, err := core.DownloadBookmark(book.URL) if err != nil { return nil, fmt.Errorf("error downloading url: %s", err) @@ -207,7 +207,7 @@ func (h *Handler) ApiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h err = json.NewDecoder(r.Body).Decode(&payload) checkError(err) - book := &model.Bookmark{ + book := &model.BookmarkDTO{ URL: payload.URL, Title: payload.Title, Excerpt: payload.Excerpt, @@ -306,7 +306,7 @@ func (h *Handler) ApiUpdateBookmark(w http.ResponseWriter, r *http.Request, ps h checkError(err) // Decode request - request := model.Bookmark{} + request := model.BookmarkDTO{} err = json.NewDecoder(r.Body).Decode(&request) checkError(err) diff --git a/internal/webserver/handler-ui.go b/internal/webserver/handler-ui.go index e281e51a3..14900f038 100644 --- a/internal/webserver/handler-ui.go +++ b/internal/webserver/handler-ui.go @@ -16,185 +16,8 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/go-shiori/warc" "github.com/julienschmidt/httprouter" - - "github.com/go-shiori/shiori/internal/model" ) -// ServeBookmarkContent is handler for GET /bookmark/:id/content -func (h *Handler) ServeBookmarkContent(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - ctx := r.Context() - - // Get bookmark ID from URL - strID := ps.ByName("id") - id, err := strconv.Atoi(strID) - checkError(err) - - // Get bookmark in database - bookmark, exist, err := h.DB.GetBookmark(ctx, id, "") - checkError(err) - - if !exist { - panic(fmt.Errorf("bookmark not found")) - } - - // If it's not public, make sure session still valid - if bookmark.Public != 1 { - err = h.validateSession(r) - if err != nil { - newPath := path.Join(h.RootPath, "/login") - redirectURL := createRedirectURL(newPath, r.URL.String()) - redirectPage(w, r, redirectURL) - return - } - } - - // Check if it has ebook. - ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") - if FileExists(ebookPath) { - bookmark.HasEbook = true - } - archivePath := fp.Join(h.DataDir, "archive", strID) - if FileExists(archivePath) { - bookmark.HasArchive = true - - // Open archive, look in cache first - var archive *warc.Archive - cacheData, found := h.ArchiveCache.Get(strID) - - if found { - archive = cacheData.(*warc.Archive) - } else { - archivePath := fp.Join(h.DataDir, "archive", strID) - archive, err = warc.Open(archivePath) - checkError(err) - - h.ArchiveCache.Set(strID, archive, 0) - } - - // Find all image and convert its source to use the archive URL. - createArchivalURL := func(archivalName string) string { - archivalURL := *r.URL - archivalURL.Path = path.Join(h.RootPath, "bookmark", strID, "archive", archivalName) - return archivalURL.String() - } - - buffer := strings.NewReader(bookmark.HTML) - doc, err := goquery.NewDocumentFromReader(buffer) - checkError(err) - - doc.Find("img, picture, figure, source").Each(func(_ int, node *goquery.Selection) { - // Get the needed attributes - src, _ := node.Attr("src") - strSrcSets, _ := node.Attr("srcset") - - // Convert `src` attributes - if src != "" { - archivalName := getArchivalName(src) - if archivalName != "" && archive.HasResource(archivalName) { - node.SetAttr("src", createArchivalURL(archivalName)) - } - } - - // Split srcset by comma, then process it like any URLs - srcSets := strings.Split(strSrcSets, ",") - for i, srcSet := range srcSets { - srcSet = strings.TrimSpace(srcSet) - parts := strings.SplitN(srcSet, " ", 2) - if parts[0] == "" { - continue - } - - archivalName := getArchivalName(parts[0]) - if archivalName != "" && archive.HasResource(archivalName) { - archivalURL := createArchivalURL(archivalName) - srcSets[i] = strings.Replace(srcSets[i], parts[0], archivalURL, 1) - } - } - - if len(srcSets) > 0 { - node.SetAttr("srcset", strings.Join(srcSets, ",")) - } - }) - - bookmark.HTML, err = goquery.OuterHtml(doc.Selection) - checkError(err) - } - - // Execute template - if developmentMode { - if err := h.PrepareTemplates(); err != nil { - log.Printf("error during template preparation: %s", err) - } - } - - tplData := struct { - RootPath string - Book model.Bookmark - }{h.RootPath, bookmark} - - err = h.templates["content"].Execute(w, &tplData) - checkError(err) -} - -// ServeThumbnailImage is handler for GET /bookmark/:id/thumb -func (h *Handler) ServeThumbnailImage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - // Get bookmark ID from URL - ctx := r.Context() - - // Get bookmark ID from URL - strID := ps.ByName("id") - - // Get bookmark from database - id, err := strconv.Atoi(strID) - checkError(err) - bookmark, exist, err := h.DB.GetBookmark(ctx, id, "") - checkError(err) - - if !exist { - log.Println("error: bookmark not found") - return - } - - // If it's not public, make sure session still valid - if bookmark.Public != 1 { - err = h.validateSession(r) - if err != nil { - newPath := path.Join(h.RootPath, "/login") - redirectURL := createRedirectURL(newPath, r.URL.String()) - redirectPage(w, r, redirectURL) - return - } - } - // Open image - imgPath := fp.Join(h.DataDir, "thumb", strID) - img, err := os.Open(imgPath) - checkError(err) - defer img.Close() - - // Get image type from its 512 first bytes - buffer := make([]byte, 512) - _, err = img.Read(buffer) - checkError(err) - - mimeType := http.DetectContentType(buffer) - w.Header().Set("Content-Type", mimeType) - - // Set cache value - info, err := img.Stat() - checkError(err) - - etag := fmt.Sprintf(`W/"%x-%x"`, info.ModTime().Unix(), info.Size()) - w.Header().Set("ETag", etag) - w.Header().Set("Cache-Control", "max-age=86400") - - // Serve image - if _, err := img.Seek(0, 0); err != nil { - log.Printf("error during image seek: %s", err) - } - _, err = io.Copy(w, img) - checkError(err) -} - // ServeBookmarkArchive is handler for GET /bookmark/:id/archive/*filepath func (h *Handler) ServeBookmarkArchive(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { ctx := r.Context() diff --git a/internal/webserver/handler.go b/internal/webserver/handler.go index 7401beae9..9e4a00245 100644 --- a/internal/webserver/handler.go +++ b/internal/webserver/handler.go @@ -6,8 +6,8 @@ import ( "net/http" "strings" - "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" cch "github.com/patrickmn/go-cache" "github.com/sirupsen/logrus" @@ -25,7 +25,7 @@ type Handler struct { ArchiveCache *cch.Cache Log bool - dependencies *config.Dependencies + dependencies *dependencies.Dependencies templates map[string]*template.Template } diff --git a/internal/webserver/server.go b/internal/webserver/server.go index 555b08317..89f0619aa 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -3,8 +3,8 @@ package webserver import ( "time" - "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" cch "github.com/patrickmn/go-cache" ) @@ -18,7 +18,7 @@ type Config struct { Log bool } -func GetLegacyHandler(cfg Config, dependencies *config.Dependencies) *Handler { +func GetLegacyHandler(cfg Config, dependencies *dependencies.Dependencies) *Handler { return &Handler{ DB: cfg.DB, DataDir: cfg.DataDir, From 4159d5e810735fa4b5e99e8d2d659d9d4e70caca Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 12 Nov 2023 10:15:27 +0100 Subject: [PATCH 02/20] new archive page --- internal/domains/filesystem.go | 24 ------ internal/http/routes/bookmark.go | 70 ++++++++-------- internal/http/server.go | 3 +- internal/model/main.go | 5 ++ internal/view/archive.html | 15 ++-- internal/view/assets/css/archive.css | 2 +- internal/view/assets/less/archive.less | 112 +++++++++++++------------ internal/webserver/handler-ui.go | 66 --------------- 8 files changed, 109 insertions(+), 188 deletions(-) delete mode 100644 internal/domains/filesystem.go diff --git a/internal/domains/filesystem.go b/internal/domains/filesystem.go deleted file mode 100644 index e28ec8da2..000000000 --- a/internal/domains/filesystem.go +++ /dev/null @@ -1,24 +0,0 @@ -package domains - -import ( - "os" - - "github.com/sirupsen/logrus" -) - -type FilesystemDomain struct { - logger *logrus.Logger - dataDir string -} - -func (d *FilesystemDomain) FileExists(path string) bool { - info, err := os.Stat(path) - return err == nil && !info.IsDir() -} - -func NewFilesystemDomain(logger *logrus.Logger, dataDir string) FilesystemDomain { - return FilesystemDomain{ - logger: logger, - dataDir: dataDir, - } -} diff --git a/internal/http/routes/bookmark.go b/internal/http/routes/bookmark.go index 397a6a60b..4e7d185d3 100644 --- a/internal/http/routes/bookmark.go +++ b/internal/http/routes/bookmark.go @@ -1,8 +1,7 @@ package routes import ( - "bytes" - "compress/gzip" + "fmt" "html/template" "net/http" "strconv" @@ -10,7 +9,6 @@ import ( fp "path/filepath" - "github.com/PuerkitoBio/goquery" "github.com/gin-gonic/gin" "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/dependencies" @@ -18,6 +16,7 @@ import ( "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" ws "github.com/go-shiori/shiori/internal/webserver" + "github.com/gofrs/uuid/v5" "github.com/sirupsen/logrus" ) @@ -28,7 +27,7 @@ type BookmarkRoutes struct { func (r *BookmarkRoutes) Setup(group *gin.RouterGroup) model.Routes { group.GET("/:id/archive", r.bookmarkArchiveHandler) - group.GET("/:id/archive/*filepath", r.bookmarkArchiveHandler) + group.GET("/:id/archive/file/*filepath", r.bookmarkArchiveFileHandler) group.GET("/:id/content", r.bookmarkContentHandler) group.GET("/:id/thumb", r.bookmarkThumbnailHandler) group.GET("/:id/ebook", r.bookmarkEbookHandler) @@ -97,6 +96,26 @@ func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) { return } + c.HTML(http.StatusOK, "archive.html", gin.H{ + "RootPath": r.deps.Config.Http.RootPath, + "Version": model.BuildVersion, + "Book": bookmark, + }) +} + +func (r *BookmarkRoutes) bookmarkArchiveFileHandler(c *gin.Context) { + ctx := context.NewContextFromGin(c) + + bookmark, err := r.getBookmark(ctx) + if err != nil { + return + } + + if !r.deps.Domains.Bookmarks.HasArchive(bookmark) { + response.SendError(c, http.StatusNotFound, nil) + return + } + resourcePath, _ := c.Params.Get("filepath") resourcePath = strings.TrimPrefix(resourcePath, "/") @@ -106,6 +125,12 @@ func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) { response.SendInternalServerError(c) return } + defer archive.Close() + + if !archive.HasResource(resourcePath) { + response.SendError(c, http.StatusNotFound, nil) + return + } content, resourceContentType, err := archive.Read(resourcePath) if err != nil { @@ -114,41 +139,12 @@ func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) { return } - // If this is HTML and root, inject shiori header - if strings.Contains(strings.ToLower(resourceContentType), "text/html") && resourcePath == "" { - // Extract gzip - buffer := bytes.NewBuffer(content) - gzipReader, err := gzip.NewReader(buffer) - if err != nil { - r.logger.WithError(err).Error("error creating gzip reader") - response.SendInternalServerError(c) - return - } - - // Parse gzipped content - doc, err := goquery.NewDocumentFromReader(gzipReader) - if err != nil { - r.logger.WithError(err).Error("error parsing gzipped content") - response.SendInternalServerError(c) - return - } - - // Revert back to HTML - outerHTML, err := goquery.OuterHtml(doc.Selection) - if err != nil { - r.logger.WithError(err).Error("error creating outer HTML") - response.SendInternalServerError(c) - return - } - - // Gzip it again and send to response writer - c.Data(http.StatusOK, "text/html", []byte(outerHTML)) - return - } + // Generate weak ETAG + shioriUUID := uuid.NewV5(uuid.NamespaceURL, model.ShioriURLNamespace) + c.Header("Etag", fmt.Sprintf("W/%s", uuid.NewV5(shioriUUID, fmt.Sprintf("%x-%x-%x", bookmark.ID, resourcePath, len(content))))) + c.Header("Cache-Control", "max-age=31536000") - // Serve content c.Header("Content-Encoding", "gzip") - // TODO: Set ETag header c.Data(http.StatusOK, resourceContentType, content) } diff --git a/internal/http/server.go b/internal/http/server.go index cc3702464..b1fa63f08 100644 --- a/internal/http/server.go +++ b/internal/http/server.go @@ -8,7 +8,6 @@ import ( "os/signal" "syscall" - "github.com/gin-contrib/gzip" "github.com/gin-contrib/requestid" "github.com/gin-gonic/gin" "github.com/go-shiori/shiori/internal/config" @@ -37,7 +36,7 @@ func (s *HttpServer) Setup(cfg *config.Config, deps *dependencies.Dependencies) templates.SetupTemplates(s.engine) - s.engine.Use(gzip.Gzip(gzip.DefaultCompression)) + // s.engine.Use(gzip.Gzip(gzip.DefaultCompression)) s.engine.Use(requestid.New()) diff --git a/internal/model/main.go b/internal/model/main.go index 694292e5b..db99a02bb 100644 --- a/internal/model/main.go +++ b/internal/model/main.go @@ -6,3 +6,8 @@ var ( BuildCommit = "none" BuildDate = "unknown" ) + +const ( + // ShioriNamespace + ShioriURLNamespace = "https://github.com/go-shiori/shiori" +) diff --git a/internal/view/archive.html b/internal/view/archive.html index bd1e1b532..6686db741 100644 --- a/internal/view/archive.html +++ b/internal/view/archive.html @@ -1,20 +1,23 @@ + $$.Book.Title$$ - + - -
+ + +
View Original - $$if .HasContent$$ - View Readable + $$if .Book.HasContent$$ + View Readable $$end$$
- + + diff --git a/internal/view/assets/css/archive.css b/internal/view/assets/css/archive.css index 043ce1c51..121d62ab3 100644 --- a/internal/view/assets/css/archive.css +++ b/internal/view/assets/css/archive.css @@ -1 +1 @@ -:root{--main:#f44336;--border:#e5e5e5;--colorLink:#999;--archiveHeaderBg:rgba(255, 255, 255, 0.95)}@media (prefers-color-scheme:dark){:root{--border:#191919;--archiveHeaderBg:rgba(41, 41, 41, 0.95)}}#shiori-archive-header{top:0;left:0;right:0;height:60px;box-sizing:border-box;position:fixed;padding:0 16px;display:flex;flex-flow:row wrap;align-items:center;font-size:16px;border-bottom:1px solid var(--border);background-color:var(--archiveHeaderBg);z-index:9999999999}#shiori-archive-header *{border-width:0;box-sizing:border-box;font-family:"Source Sans Pro",sans-serif;margin:0;padding:0}#shiori-archive-header>:not(:last-child){margin-right:8px}#shiori-archive-header>.spacer{flex:1}#shiori-archive-header #shiori-logo{font-size:2em;font-weight:100;color:var(--main)}#shiori-archive-header #shiori-logo span{margin-right:8px}#shiori-archive-header a{display:block;color:var(--colorLink);text-decoration:underline}#shiori-archive-header a:focus,#shiori-archive-header a:hover{color:var(--main)}@media (max-width:600px){#shiori-archive-header{font-size:14px;height:50px}#shiori-archive-header #shiori-logo{font-size:1.5em}}body.shiori-archive-content{margin-top:60px!important}@media (max-width:600px){body.shiori-archive-content{margin-top:50px!important}} \ No newline at end of file +:root{--main:#f44336;--border:#e5e5e5;--colorLink:#999;--archiveHeaderBg:rgba(255, 255, 255, 0.95)}@media (prefers-color-scheme:dark){:root{--border:#191919;--archiveHeaderBg:rgba(41, 41, 41, 0.95)}}body{padding:0;margin:0}*{box-sizing:border-box}body.archive{display:grid;grid-template-rows:minmax(1px,auto) 1fr;height:100vh;width:100vw}body.archive .header{display:flex;flex-flow:row wrap;height:60px;box-sizing:border-box;padding:0 16px;align-items:center;font-size:16px;border-bottom:1px solid var(--border);background-color:var(--archiveHeaderBg)}body.archive .header *{border-width:0;box-sizing:border-box;font-family:"Source Sans Pro",sans-serif;margin:0;padding:0}body.archive .header>:not(:last-child){margin-right:8px}body.archive .header>.spacer{flex:1}body.archive .header #shiori-logo{font-size:2em;font-weight:100;color:var(--main)}body.archive .header #shiori-logo span{margin-right:8px}body.archive .header a{display:block;color:var(--colorLink);text-decoration:underline}body.archive .header a:focus,body.archive .header a:hover{color:var(--main)}@media (max-width:600px){body.archive .header{font-size:14px;height:50px}body.archive .header #shiori-logo{font-size:1.5em}}body.archive iframe{width:100%;height:100%} \ No newline at end of file diff --git a/internal/view/assets/less/archive.less b/internal/view/assets/less/archive.less index 30107acb2..1e00ba3dd 100644 --- a/internal/view/assets/less/archive.less +++ b/internal/view/assets/less/archive.less @@ -14,73 +14,81 @@ @screen-sm-max: 600px; @screen-sm-header-height: 50px; -#shiori-archive-header { - top: 0; - left: 0; - right: 0; - height: @header-height; +body { + padding: 0; + margin: 0; +} + +* { box-sizing: border-box; - position: fixed; - padding: 0 16px; - display: flex; - flex-flow: row wrap; - align-items: center; - font-size: 16px; - border-bottom: 1px solid var(--border); - background-color: var(--archiveHeaderBg); - z-index: 9999999999; - - * { - border-width: 0; - box-sizing: border-box; - font-family: "Source Sans Pro", sans-serif; - margin: 0; - padding: 0; - } +} - > *:not(:last-child) { - margin-right: 8px; - } +body.archive { + display: grid; + grid-template-rows: minmax(1px, auto) 1fr; + height: 100vh; + width: 100vw; - > .spacer { - flex: 1; - } + .header { + display: flex; + flex-flow: row wrap; + height: @header-height; + box-sizing: border-box; + padding: 0 16px; + align-items: center; + font-size: 16px; + border-bottom: 1px solid var(--border); + background-color: var(--archiveHeaderBg); - #shiori-logo { - font-size: 2em; - font-weight: 100; - color: var(--main); + * { + border-width: 0; + box-sizing: border-box; + font-family: "Source Sans Pro", sans-serif; + margin: 0; + padding: 0; + } - span { + > *:not(:last-child) { margin-right: 8px; } - } - a { - display: block; - color: var(--colorLink); - text-decoration: underline; + > .spacer { + flex: 1; + } - &:hover, - &:focus { + #shiori-logo { + font-size: 2em; + font-weight: 100; color: var(--main); + + span { + margin-right: 8px; + } } - } - @media (max-width: @screen-sm-max) { - font-size: 14px; - height: @screen-sm-header-height; + a { + display: block; + color: var(--colorLink); + text-decoration: underline; - #shiori-logo { - font-size: 1.5em; + &:hover, + &:focus { + color: var(--main); + } } - } -} -body.shiori-archive-content { - margin-top: @header-height !important; + @media (max-width: @screen-sm-max) { + font-size: 14px; + height: @screen-sm-header-height; + + #shiori-logo { + font-size: 1.5em; + } + } + } - @media (max-width: @screen-sm-max) { - margin-top: @screen-sm-header-height !important; + iframe { + width: 100%; + height: 100%; } } diff --git a/internal/webserver/handler-ui.go b/internal/webserver/handler-ui.go index 14900f038..998a12476 100644 --- a/internal/webserver/handler-ui.go +++ b/internal/webserver/handler-ui.go @@ -4,10 +4,8 @@ import ( "bytes" "compress/gzip" "fmt" - "io" "log" "net/http" - "os" "path" fp "path/filepath" "strconv" @@ -112,67 +110,3 @@ func (h *Handler) ServeBookmarkArchive(w http.ResponseWriter, r *http.Request, p log.Printf("error writing response: %s", err) } } - -// serveEbook is handler for GET /bookmark/:id/ebook -func (h *Handler) ServeBookmarkEbook(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - ctx := r.Context() - - // Get bookmark ID from URL - strID := ps.ByName("id") - id, err := strconv.Atoi(strID) - checkError(err) - // Get bookmark in database - bookmark, exist, err := h.DB.GetBookmark(ctx, id, "") - checkError(err) - - if !exist { - http.Error(w, "bookmark not found", http.StatusNotFound) - return - } - - // If it's not public, make sure session still valid - if bookmark.Public != 1 { - err = h.validateSession(r) - if err != nil { - newPath := path.Join(h.RootPath, "/login") - redirectURL := createRedirectURL(newPath, r.URL.String()) - redirectPage(w, r, redirectURL) - return - } - } - - // Check if it has ebook. - ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") - if !FileExists(ebookPath) { - http.Error(w, "ebook not found", http.StatusNotFound) - return - } - - epub, err := os.Open(ebookPath) - if err != nil { - http.Error(w, "Internal server error", http.StatusNotFound) - return - } - defer epub.Close() - // Set content type - w.Header().Set("Content-Type", "application/epub+zip") - // Set cache value - info, err := epub.Stat() - if err != nil { - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } - etag := fmt.Sprintf(`W/"%x-%x"`, info.ModTime().Unix(), info.Size()) - w.Header().Set("ETag", etag) - w.Header().Set("Cache-Control", "max-age=86400") - - // Serve epub file - if _, err := epub.Seek(0, 0); err != nil { - log.Printf("error during epub seek: %s", err) - } - _, err = io.Copy(w, epub) - if err != nil { - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } -} From 1b49d90ede7eaa7898012721cbb21cc68fa2179d Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sat, 18 Nov 2023 10:24:04 +0100 Subject: [PATCH 03/20] remove unused go generate comment --- main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/main.go b/main.go index 037b73ea4..400f9b355 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,3 @@ -//go:generate go run assets-generator.go - package main import ( From bb84c19c47d794e8679db3b38f89e8b1e9695062 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sat, 18 Nov 2023 10:24:44 +0100 Subject: [PATCH 04/20] database mock --- Makefile | 5 + go.mod | 1 + internal/database/database.go | 1 + internal/mocks/database.go | 266 ++++++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+) create mode 100644 internal/mocks/database.go diff --git a/Makefile b/Makefile index bafba3f27..676d1cee6 100644 --- a/Makefile +++ b/Makefile @@ -94,3 +94,8 @@ build: clean coverage: $(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt ./... $(GO) tool cover -html=coverage.txt + +## Run generate accross the project +.PHONY: generated +generate: + $(GO) generate ./... diff --git a/go.mod b/go.mod index e20c1d159..d2cf4d778 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.2 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f + go.uber.org/mock v0.3.0 golang.org/x/crypto v0.15.0 golang.org/x/image v0.14.0 golang.org/x/net v0.18.0 diff --git a/internal/database/database.go b/internal/database/database.go index d79c56ca0..6bf210ac0 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination=../mocks/database.go -package=mocks github.com/go-shiori/shiori/internal/database DB package database import ( diff --git a/internal/mocks/database.go b/internal/mocks/database.go new file mode 100644 index 000000000..6f0fc04fe --- /dev/null +++ b/internal/mocks/database.go @@ -0,0 +1,266 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/go-shiori/shiori/internal/database (interfaces: DB) +// +// Generated by this command: +// +// mockgen -destination=../mocks/database.go -package=mocks github.com/go-shiori/shiori/internal/database DB +// +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + database "github.com/go-shiori/shiori/internal/database" + model "github.com/go-shiori/shiori/internal/model" + gomock "go.uber.org/mock/gomock" +) + +// MockDB is a mock of DB interface. +type MockDB struct { + ctrl *gomock.Controller + recorder *MockDBMockRecorder +} + +// MockDBMockRecorder is the mock recorder for MockDB. +type MockDBMockRecorder struct { + mock *MockDB +} + +// NewMockDB creates a new mock instance. +func NewMockDB(ctrl *gomock.Controller) *MockDB { + mock := &MockDB{ctrl: ctrl} + mock.recorder = &MockDBMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDB) EXPECT() *MockDBMockRecorder { + return m.recorder +} + +// CreateTags mocks base method. +func (m *MockDB) CreateTags(arg0 context.Context, arg1 ...model.Tag) error { + m.ctrl.T.Helper() + varargs := []any{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CreateTags", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateTags indicates an expected call of CreateTags. +func (mr *MockDBMockRecorder) CreateTags(arg0 any, arg1 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTags", reflect.TypeOf((*MockDB)(nil).CreateTags), varargs...) +} + +// DeleteAccounts mocks base method. +func (m *MockDB) DeleteAccounts(arg0 context.Context, arg1 ...string) error { + m.ctrl.T.Helper() + varargs := []any{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteAccounts", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAccounts indicates an expected call of DeleteAccounts. +func (mr *MockDBMockRecorder) DeleteAccounts(arg0 any, arg1 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccounts", reflect.TypeOf((*MockDB)(nil).DeleteAccounts), varargs...) +} + +// DeleteBookmarks mocks base method. +func (m *MockDB) DeleteBookmarks(arg0 context.Context, arg1 ...int) error { + m.ctrl.T.Helper() + varargs := []any{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteBookmarks", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBookmarks indicates an expected call of DeleteBookmarks. +func (mr *MockDBMockRecorder) DeleteBookmarks(arg0 any, arg1 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBookmarks", reflect.TypeOf((*MockDB)(nil).DeleteBookmarks), varargs...) +} + +// GetAccount mocks base method. +func (m *MockDB) GetAccount(arg0 context.Context, arg1 string) (model.Account, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAccount", arg0, arg1) + ret0, _ := ret[0].(model.Account) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetAccount indicates an expected call of GetAccount. +func (mr *MockDBMockRecorder) GetAccount(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockDB)(nil).GetAccount), arg0, arg1) +} + +// GetAccounts mocks base method. +func (m *MockDB) GetAccounts(arg0 context.Context, arg1 database.GetAccountsOptions) ([]model.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAccounts", arg0, arg1) + ret0, _ := ret[0].([]model.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAccounts indicates an expected call of GetAccounts. +func (mr *MockDBMockRecorder) GetAccounts(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccounts", reflect.TypeOf((*MockDB)(nil).GetAccounts), arg0, arg1) +} + +// GetBookmark mocks base method. +func (m *MockDB) GetBookmark(arg0 context.Context, arg1 int, arg2 string) (model.BookmarkDTO, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBookmark", arg0, arg1, arg2) + ret0, _ := ret[0].(model.BookmarkDTO) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetBookmark indicates an expected call of GetBookmark. +func (mr *MockDBMockRecorder) GetBookmark(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBookmark", reflect.TypeOf((*MockDB)(nil).GetBookmark), arg0, arg1, arg2) +} + +// GetBookmarks mocks base method. +func (m *MockDB) GetBookmarks(arg0 context.Context, arg1 database.GetBookmarksOptions) ([]model.BookmarkDTO, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBookmarks", arg0, arg1) + ret0, _ := ret[0].([]model.BookmarkDTO) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBookmarks indicates an expected call of GetBookmarks. +func (mr *MockDBMockRecorder) GetBookmarks(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBookmarks", reflect.TypeOf((*MockDB)(nil).GetBookmarks), arg0, arg1) +} + +// GetBookmarksCount mocks base method. +func (m *MockDB) GetBookmarksCount(arg0 context.Context, arg1 database.GetBookmarksOptions) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBookmarksCount", arg0, arg1) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBookmarksCount indicates an expected call of GetBookmarksCount. +func (mr *MockDBMockRecorder) GetBookmarksCount(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBookmarksCount", reflect.TypeOf((*MockDB)(nil).GetBookmarksCount), arg0, arg1) +} + +// GetTags mocks base method. +func (m *MockDB) GetTags(arg0 context.Context) ([]model.Tag, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTags", arg0) + ret0, _ := ret[0].([]model.Tag) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTags indicates an expected call of GetTags. +func (mr *MockDBMockRecorder) GetTags(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTags", reflect.TypeOf((*MockDB)(nil).GetTags), arg0) +} + +// Migrate mocks base method. +func (m *MockDB) Migrate() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Migrate") + ret0, _ := ret[0].(error) + return ret0 +} + +// Migrate indicates an expected call of Migrate. +func (mr *MockDBMockRecorder) Migrate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Migrate", reflect.TypeOf((*MockDB)(nil).Migrate)) +} + +// RenameTag mocks base method. +func (m *MockDB) RenameTag(arg0 context.Context, arg1 int, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RenameTag", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RenameTag indicates an expected call of RenameTag. +func (mr *MockDBMockRecorder) RenameTag(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenameTag", reflect.TypeOf((*MockDB)(nil).RenameTag), arg0, arg1, arg2) +} + +// SaveAccount mocks base method. +func (m *MockDB) SaveAccount(arg0 context.Context, arg1 model.Account) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveAccount", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveAccount indicates an expected call of SaveAccount. +func (mr *MockDBMockRecorder) SaveAccount(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveAccount", reflect.TypeOf((*MockDB)(nil).SaveAccount), arg0, arg1) +} + +// SaveAccountSettings mocks base method. +func (m *MockDB) SaveAccountSettings(arg0 context.Context, arg1 model.Account) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveAccountSettings", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveAccountSettings indicates an expected call of SaveAccountSettings. +func (mr *MockDBMockRecorder) SaveAccountSettings(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveAccountSettings", reflect.TypeOf((*MockDB)(nil).SaveAccountSettings), arg0, arg1) +} + +// SaveBookmarks mocks base method. +func (m *MockDB) SaveBookmarks(arg0 context.Context, arg1 bool, arg2 ...model.BookmarkDTO) ([]model.BookmarkDTO, error) { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "SaveBookmarks", varargs...) + ret0, _ := ret[0].([]model.BookmarkDTO) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SaveBookmarks indicates an expected call of SaveBookmarks. +func (mr *MockDBMockRecorder) SaveBookmarks(arg0, arg1 any, arg2 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveBookmarks", reflect.TypeOf((*MockDB)(nil).SaveBookmarks), varargs...) +} From 600e74192705809f58b12122d83862569cebf0fa Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sat, 18 Nov 2023 10:24:55 +0100 Subject: [PATCH 05/20] utils cleanup --- internal/webserver/utils.go | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/internal/webserver/utils.go b/internal/webserver/utils.go index 3c7fdcaf8..39fd8ff5c 100644 --- a/internal/webserver/utils.go +++ b/internal/webserver/utils.go @@ -7,13 +7,9 @@ import ( "net/http" nurl "net/url" "os" - "regexp" - "strings" "syscall" ) -var rxRepeatedStrip = regexp.MustCompile(`(?i)-+`) - func createRedirectURL(newPath, previousPath string) string { urlQueries := nurl.Values{} urlQueries.Set("dst", previousPath) @@ -53,37 +49,6 @@ func createTemplate(filename string, funcMap template.FuncMap) (*template.Templa return template.New(filename).Delims("$$", "$$").Funcs(funcMap).Parse(string(srcContent)) } -// getArchivalName converts an URL into an archival name. -func getArchivalName(src string) string { - archivalURL := src - - // Some URL have its query or path escaped, e.g. Wikipedia and Dev.to. - // For example, Wikipedia's stylesheet looks like this : - // load.php?lang=en&modules=ext.3d.styles%7Cext.cite.styles%7Cext.uls.interlanguage - // However, when browser download it, it will be registered as unescaped query : - // load.php?lang=en&modules=ext.3d.styles|ext.cite.styles|ext.uls.interlanguage - // So, for archival URL, we need to unescape the query and path first. - tmp, err := nurl.Parse(src) - if err == nil { - unescapedQuery, _ := nurl.QueryUnescape(tmp.RawQuery) - if unescapedQuery != "" { - tmp.RawQuery = unescapedQuery - } - - archivalURL = tmp.String() - archivalURL = strings.Replace(archivalURL, tmp.EscapedPath(), tmp.Path, 1) - } - - archivalURL = strings.ReplaceAll(archivalURL, "://", "/") - archivalURL = strings.ReplaceAll(archivalURL, "?", "-") - archivalURL = strings.ReplaceAll(archivalURL, "#", "-") - archivalURL = strings.ReplaceAll(archivalURL, "/", "-") - archivalURL = strings.ReplaceAll(archivalURL, " ", "-") - archivalURL = rxRepeatedStrip.ReplaceAllString(archivalURL, "-") - - return archivalURL -} - func checkError(err error) { if err == nil { return From 887d23fab2115c3deb0889157cef7ab837f0b4e4 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sat, 18 Nov 2023 10:25:17 +0100 Subject: [PATCH 06/20] unused var --- internal/webserver/handler.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/webserver/handler.go b/internal/webserver/handler.go index 9e4a00245..e3e6e3b1d 100644 --- a/internal/webserver/handler.go +++ b/internal/webserver/handler.go @@ -13,8 +13,6 @@ import ( "github.com/sirupsen/logrus" ) -var developmentMode = false - // Handler is Handler for serving the web interface. type Handler struct { DB database.DB From 68f12efb33a486133da999da0f031a4d2eb397dd Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sat, 18 Nov 2023 10:26:12 +0100 Subject: [PATCH 07/20] domains refactor and tests --- go.mod | 2 +- go.sum | 5 ++ internal/cmd/root.go | 7 ++- internal/dependencies/dependencies.go | 15 +++-- internal/domains/accounts.go | 23 +++---- internal/domains/archiver.go | 16 +++-- internal/domains/bookmarks.go | 36 +++++------ internal/domains/bookmarks_test.go | 91 +++++++++++++++++++++++++++ internal/domains/storage.go | 32 ++++++++++ internal/domains/storage_test.go | 47 ++++++++++++++ internal/model/domains.go | 34 ++++++++++ internal/testutil/shiori.go | 7 ++- 12 files changed, 260 insertions(+), 55 deletions(-) create mode 100644 internal/domains/bookmarks_test.go create mode 100644 internal/domains/storage.go create mode 100644 internal/domains/storage_test.go create mode 100644 internal/model/domains.go diff --git a/go.mod b/go.mod index d2cf4d778..1409c5eb4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/disintegration/imaging v1.6.2 github.com/fatih/color v1.16.0 - github.com/gin-contrib/gzip v0.0.6 github.com/gin-contrib/requestid v0.0.6 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.9.1 @@ -24,6 +23,7 @@ require ( github.com/muesli/go-app-paths v0.2.2 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 + github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e github.com/sethvargo/go-envconfig v0.9.0 github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index bd444ba2e..0cb12a3cf 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,7 @@ github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8 github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -205,6 +206,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e h1:51xcRlSMBU5rhM9KahnJGfEsBPVPz3182TgFRowA8yY= +github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -269,6 +272,8 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= +go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= diff --git a/internal/cmd/root.go b/internal/cmd/root.go index a2e86b3e8..113074e4c 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -98,9 +98,10 @@ func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *depen } dependencies := dependencies.NewDependencies(logger, db, cfg) - dependencies.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db) - dependencies.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir) - dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(logger, db, cfg) + dependencies.Domains.Auth = domains.NewAccountsDomain(dependencies) + dependencies.Domains.Archiver = domains.NewArchiverDomain(dependencies) + dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(dependencies) + dependencies.Domains.Storage = domains.NewStorageDomain(dependencies, os.DirFS(cfg.Storage.DataDir)) // Workaround: Get accounts to make sure at least one is present in the database. // If there's no accounts in the database, create the shiori/gopher account the legacy api diff --git a/internal/dependencies/dependencies.go b/internal/dependencies/dependencies.go index aa824b410..b51152e40 100644 --- a/internal/dependencies/dependencies.go +++ b/internal/dependencies/dependencies.go @@ -3,19 +3,22 @@ package dependencies import ( "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/database" - "github.com/go-shiori/shiori/internal/domains" + "github.com/go-shiori/shiori/internal/model" "github.com/sirupsen/logrus" ) +type Domains struct { + Archiver model.ArchiverDomain + Auth model.AccountsDomain + Bookmarks model.BookmarksDomain + Storage model.StorageDomain +} + type Dependencies struct { Log *logrus.Logger Database database.DB Config *config.Config - Domains struct { - Auth domains.AccountsDomain - Archiver domains.ArchiverDomain - Bookmarks domains.BookmarksDomain - } + Domains *Domains } func NewDependencies(log *logrus.Logger, db database.DB, cfg *config.Config) *Dependencies { diff --git a/internal/domains/accounts.go b/internal/domains/accounts.go index 132fa1b27..3607c7bd5 100644 --- a/internal/domains/accounts.go +++ b/internal/domains/accounts.go @@ -5,18 +5,15 @@ import ( "fmt" "time" - "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/golang-jwt/jwt/v5" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/crypto/bcrypt" ) type AccountsDomain struct { - logger *logrus.Logger - db database.DB - secret []byte + deps *dependencies.Dependencies } type JWTClaim struct { @@ -32,7 +29,7 @@ func (d *AccountsDomain) CheckToken(ctx context.Context, userJWT string) (*model return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) } - return d.secret, nil + return d.deps.Config.Http.SecretKey, nil }) if err != nil { return nil, errors.Wrap(err, "error parsing token") @@ -52,7 +49,7 @@ func (d *AccountsDomain) CheckToken(ctx context.Context, userJWT string) (*model } func (d *AccountsDomain) GetAccountFromCredentials(ctx context.Context, username, password string) (*model.Account, error) { - account, _, err := d.db.GetAccount(ctx, username) + account, _, err := d.deps.Database.GetAccount(ctx, username) if err != nil { return nil, fmt.Errorf("username and password do not match") } @@ -72,18 +69,16 @@ func (d *AccountsDomain) CreateTokenForAccount(account *model.Account, expiratio token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - t, err := token.SignedString(d.secret) + t, err := token.SignedString(d.deps.Config.Http.SecretKey) if err != nil { - d.logger.WithError(err).Error("error signing token") + d.deps.Log.WithError(err).Error("error signing token") } return t, err } -func NewAccountsDomain(logger *logrus.Logger, secretKey string, db database.DB) AccountsDomain { - return AccountsDomain{ - logger: logger, - db: db, - secret: []byte(secretKey), +func NewAccountsDomain(deps *dependencies.Dependencies) *AccountsDomain { + return &AccountsDomain{ + deps: deps, } } diff --git a/internal/domains/archiver.go b/internal/domains/archiver.go index 706eedf8c..e2488d094 100644 --- a/internal/domains/archiver.go +++ b/internal/domains/archiver.go @@ -6,14 +6,13 @@ import ( "strconv" "github.com/go-shiori/shiori/internal/core" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/go-shiori/warc" - "github.com/sirupsen/logrus" ) type ArchiverDomain struct { - dataDir string - logger *logrus.Logger + deps *dependencies.Dependencies } func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model.BookmarkDTO, error) { @@ -23,7 +22,7 @@ func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model } processRequest := core.ProcessRequest{ - DataDir: d.dataDir, + DataDir: d.deps.Config.Storage.DataDir, Bookmark: book, Content: content, ContentType: contentType, @@ -40,7 +39,7 @@ func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model } func (d *ArchiverDomain) GetBookmarkArchive(book *model.BookmarkDTO) (*warc.Archive, error) { - archivePath := filepath.Join(d.dataDir, "archive", strconv.Itoa(book.ID)) + archivePath := filepath.Join(d.deps.Config.Storage.DataDir, "archive", strconv.Itoa(book.ID)) if !FileExists(archivePath) { return nil, fmt.Errorf("archive not found") @@ -49,9 +48,8 @@ func (d *ArchiverDomain) GetBookmarkArchive(book *model.BookmarkDTO) (*warc.Arch return warc.Open(archivePath) } -func NewArchiverDomain(logger *logrus.Logger, dataDir string) ArchiverDomain { - return ArchiverDomain{ - dataDir: dataDir, - logger: logger, +func NewArchiverDomain(deps *dependencies.Dependencies) *ArchiverDomain { + return &ArchiverDomain{ + deps: deps, } } diff --git a/internal/domains/bookmarks.go b/internal/domains/bookmarks.go index 4d17fdf3e..c18529850 100644 --- a/internal/domains/bookmarks.go +++ b/internal/domains/bookmarks.go @@ -11,39 +11,35 @@ import ( "strings" "github.com/PuerkitoBio/goquery" - "github.com/go-shiori/shiori/internal/config" - "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/go-shiori/warc" - "github.com/sirupsen/logrus" ) type BookmarksDomain struct { - logger *logrus.Logger - db database.DB - cfg *config.Config + deps *dependencies.Dependencies } func (d *BookmarksDomain) HasEbook(b *model.BookmarkDTO) bool { - ebookPath := filepath.Join(d.cfg.Storage.DataDir, "ebook", strconv.Itoa(b.ID)+".epub") - return FileExists(ebookPath) + ebookPath := filepath.Join("ebook", strconv.Itoa(b.ID)+".epub") + return d.deps.Domains.Storage.FileExists(ebookPath) } func (d *BookmarksDomain) HasArchive(b *model.BookmarkDTO) bool { - archivePath := filepath.Join(d.cfg.Storage.DataDir, "archive", strconv.Itoa(b.ID)) - return FileExists(archivePath) + archivePath := filepath.Join(d.deps.Config.Storage.DataDir, "archive", strconv.Itoa(b.ID)) + return d.deps.Domains.Storage.FileExists(archivePath) } func (d *BookmarksDomain) GetThumbnailPath(b *model.BookmarkDTO) string { - return filepath.Join(d.cfg.Storage.DataDir, "thumb", strconv.Itoa(b.ID)) + return filepath.Join("thumb", strconv.Itoa(b.ID)) } func (d *BookmarksDomain) HasThumbnail(b *model.BookmarkDTO) bool { - return FileExists(d.GetThumbnailPath(b)) + return d.deps.Domains.Storage.FileExists(d.GetThumbnailPath(b)) } func (d *BookmarksDomain) GetBookmark(ctx context.Context, id model.DBID) (*model.BookmarkDTO, error) { - bookmark, _, err := d.db.GetBookmark(ctx, int(id), "") + bookmark, _, err := d.deps.Database.GetBookmark(ctx, int(id), "") if err != nil { return nil, fmt.Errorf("failed to get bookmark: %w", err) } @@ -63,7 +59,9 @@ func (d *BookmarksDomain) GetBookmarkContentsFromArchive(bookmark *model.Bookmar } // Open archive, look in cache first - archivePath := filepath.Join(d.cfg.Storage.DataDir, "archive", fmt.Sprintf("%d", bookmark.ID)) + archivePath := filepath.Join(d.deps.Config.Storage.DataDir, "archive", fmt.Sprintf("%d", bookmark.ID)) + // TODO: Move to archiver domain + // TODO: Use storagedomain to operate with the file archive, err := warc.Open(archivePath) if err != nil { return "", fmt.Errorf("failed to open archive: %w", err) @@ -72,7 +70,7 @@ func (d *BookmarksDomain) GetBookmarkContentsFromArchive(bookmark *model.Bookmar // Find all image and convert its source to use the archive URL. createArchivalURL := func(archivalName string) string { var archivalURL url.URL - archivalURL.Path = path.Join(d.cfg.Http.RootPath, "bookmark", fmt.Sprintf("%d", bookmark.ID), "archive", archivalName) + archivalURL.Path = path.Join(d.deps.Config.Http.RootPath, "bookmark", fmt.Sprintf("%d", bookmark.ID), "archive", archivalName) return archivalURL.String() } @@ -124,10 +122,8 @@ func (d *BookmarksDomain) GetBookmarkContentsFromArchive(bookmark *model.Bookmar return template.HTML(bookmark.HTML), nil } -func NewBookmarksDomain(logger *logrus.Logger, db database.DB, cfg *config.Config) BookmarksDomain { - return BookmarksDomain{ - logger: logger, - db: db, - cfg: cfg, +func NewBookmarksDomain(deps *dependencies.Dependencies) *BookmarksDomain { + return &BookmarksDomain{ + deps: deps, } } diff --git a/internal/domains/bookmarks_test.go b/internal/domains/bookmarks_test.go new file mode 100644 index 000000000..1523a8667 --- /dev/null +++ b/internal/domains/bookmarks_test.go @@ -0,0 +1,91 @@ +package domains_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" + "github.com/go-shiori/shiori/internal/domains" + "github.com/go-shiori/shiori/internal/mocks" + "github.com/go-shiori/shiori/internal/model" + "github.com/psanford/memfs" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestBookmarkDomain(t *testing.T) { + fs := memfs.New() + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + deps := &dependencies.Dependencies{ + Database: mocks.NewMockDB(mockCtrl), + Config: config.ParseServerConfiguration(context.TODO(), logrus.New()), + Log: logrus.New(), + Domains: &dependencies.Domains{}, + } + deps.Domains.Storage = domains.NewStorageDomain(deps, fs) + + fs.MkdirAll("thumb", 0755) + fs.WriteFile("thumb/1", []byte("x"), 0644) + fs.MkdirAll("ebook", 0755) + fs.WriteFile("ebook/1.epub", []byte("x"), 0644) + fs.MkdirAll("archive", 0755) + // TODO: write a valid archive file + fs.WriteFile("archive/1", []byte("x"), 0644) + + domain := domains.NewBookmarksDomain(deps) + t.Run("HasEbook", func(t *testing.T) { + t.Run("Yes", func(t *testing.T) { + require.True(t, domain.HasEbook(&model.BookmarkDTO{ID: 1})) + }) + t.Run("No", func(t *testing.T) { + require.False(t, domain.HasEbook(&model.BookmarkDTO{ID: 2})) + }) + }) + + t.Run("HasArchive", func(t *testing.T) { + t.Run("Yes", func(t *testing.T) { + require.True(t, domain.HasArchive(&model.BookmarkDTO{ID: 1})) + }) + t.Run("No", func(t *testing.T) { + require.False(t, domain.HasArchive(&model.BookmarkDTO{ID: 2})) + }) + }) + + t.Run("GetThumbnailPath", func(t *testing.T) { + thumbnailPath := domain.GetThumbnailPath(&model.BookmarkDTO{ID: 1}) + require.Equal(t, filepath.Join("thumb/1"), thumbnailPath) + }) + + t.Run("HasThumbnail", func(t *testing.T) { + t.Run("Yes", func(t *testing.T) { + require.True(t, domain.HasThumbnail(&model.BookmarkDTO{ID: 1})) + }) + t.Run("No", func(t *testing.T) { + require.False(t, domain.HasThumbnail(&model.BookmarkDTO{ID: 2})) + }) + }) + + t.Run("GetBookmark", func(t *testing.T) { + t.Run("Success", func(t *testing.T) { + deps.Database.(*mocks.MockDB).EXPECT(). + GetBookmark(gomock.Any(), 1, ""). + Return(model.BookmarkDTO{ + ID: 1, + HTML: "

hello world

", + }, true, nil) + bookmark, err := domain.GetBookmark(context.Background(), 1) + require.NoError(t, err) + require.Equal(t, 1, bookmark.ID) + + // Check DTO attributes + require.True(t, bookmark.HasEbook) + require.True(t, bookmark.HasArchive) + }) + }) +} diff --git a/internal/domains/storage.go b/internal/domains/storage.go new file mode 100644 index 000000000..b39e817d1 --- /dev/null +++ b/internal/domains/storage.go @@ -0,0 +1,32 @@ +package domains + +import ( + "io/fs" + + "github.com/go-shiori/shiori/internal/dependencies" +) + +type StorageDomain struct { + deps *dependencies.Dependencies + fs fs.FS +} + +// FileExists checks if a file exists in storage. +func (d *StorageDomain) FileExists(name string) bool { + info, err := fs.Stat(d.fs, name) + return err == nil && !info.IsDir() +} + +// DirExists checks if a directory exists in storage. +func (d *StorageDomain) DirExists(name string) bool { + info, err := fs.Stat(d.fs, name) + d.deps.Log.Info(info) + return err == nil && info.IsDir() +} + +func NewStorageDomain(deps *dependencies.Dependencies, fs fs.FS) *StorageDomain { + return &StorageDomain{ + deps: deps, + fs: fs, + } +} diff --git a/internal/domains/storage_test.go b/internal/domains/storage_test.go new file mode 100644 index 000000000..ee913faf0 --- /dev/null +++ b/internal/domains/storage_test.go @@ -0,0 +1,47 @@ +package domains_test + +import ( + "context" + "testing" + + "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/dependencies" + "github.com/go-shiori/shiori/internal/domains" + "github.com/psanford/memfs" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestDirExists(t *testing.T) { + fs := memfs.New() + fs.MkdirAll("foo", 0755) + + domain := domains.NewStorageDomain( + &dependencies.Dependencies{ + Config: config.ParseServerConfiguration(context.TODO(), logrus.New()), + Log: logrus.New(), + }, + fs, + ) + + require.True(t, domain.DirExists("foo")) + require.False(t, domain.DirExists("foo/file")) + require.False(t, domain.DirExists("bar")) +} + +func TestFileExists(t *testing.T) { + fs := memfs.New() + fs.MkdirAll("foo", 0755) + fs.WriteFile("foo/file", []byte("hello world"), 0644) + + domain := domains.NewStorageDomain( + &dependencies.Dependencies{ + Config: config.ParseServerConfiguration(context.TODO(), logrus.New()), + Log: logrus.New(), + }, + fs, + ) + + require.True(t, domain.FileExists("foo/file")) + require.False(t, domain.FileExists("bar")) +} diff --git a/internal/model/domains.go b/internal/model/domains.go new file mode 100644 index 000000000..87a7d00aa --- /dev/null +++ b/internal/model/domains.go @@ -0,0 +1,34 @@ +package model + +import ( + "context" + "html/template" + "time" + + "github.com/go-shiori/warc" +) + +type BookmarksDomain interface { + HasEbook(b *BookmarkDTO) bool + HasArchive(b *BookmarkDTO) bool + GetThumbnailPath(b *BookmarkDTO) string + HasThumbnail(b *BookmarkDTO) bool + GetBookmark(ctx context.Context, id DBID) (*BookmarkDTO, error) + GetBookmarkContentsFromArchive(bookmark *BookmarkDTO) (template.HTML, error) +} + +type AccountsDomain interface { + CheckToken(ctx context.Context, userJWT string) (*Account, error) + GetAccountFromCredentials(ctx context.Context, username, password string) (*Account, error) + CreateTokenForAccount(account *Account, expiration time.Time) (string, error) +} + +type ArchiverDomain interface { + DownloadBookmarkArchive(book BookmarkDTO) (*BookmarkDTO, error) + GetBookmarkArchive(book *BookmarkDTO) (*warc.Archive, error) +} + +type StorageDomain interface { + FileExists(path string) bool + DirExists(path string) bool +} diff --git a/internal/testutil/shiori.go b/internal/testutil/shiori.go index 479e5f914..48d88cf77 100644 --- a/internal/testutil/shiori.go +++ b/internal/testutil/shiori.go @@ -9,6 +9,7 @@ import ( "github.com/go-shiori/shiori/internal/database" "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/domains" + "github.com/psanford/memfs" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -31,8 +32,10 @@ func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logg deps := dependencies.NewDependencies(logger, db, cfg) deps.Database = db - deps.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db) - deps.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir) + deps.Domains.Auth = domains.NewAccountsDomain(deps) + deps.Domains.Archiver = domains.NewArchiverDomain(deps) + deps.Domains.Bookmarks = domains.NewBookmarksDomain(deps) + deps.Domains.Storage = domains.NewStorageDomain(deps, memfs.New()) return cfg, deps } From 939f394b59150265bc75ffce2f133f290d81ec87 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 09:38:03 +0100 Subject: [PATCH 08/20] fixed secret key type --- internal/cmd/server.go | 2 +- internal/config/config.go | 6 +++--- internal/dependencies/dependencies.go | 1 + internal/testutil/shiori.go | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 53818ac8c..aa602afce 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -37,7 +37,7 @@ func newServerCommandHandler() func(cmd *cobra.Command, args []string) { rootPath, _ := cmd.Flags().GetString("webroot") accessLog, _ := cmd.Flags().GetBool("access-log") serveWebUI, _ := cmd.Flags().GetBool("serve-web-ui") - secretKey, _ := cmd.Flags().GetString("secret-key") + secretKey, _ := cmd.Flags().GetBytesHex("secret-key") cfg, dependencies := initShiori(ctx, cmd) diff --git a/internal/config/config.go b/internal/config/config.go index ca46a6d4b..de6c72b18 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -50,7 +50,7 @@ type HttpConfig struct { RootPath string `env:"HTTP_ROOT_PATH,default=/"` AccessLog bool `env:"HTTP_ACCESS_LOG,default=True"` ServeWebUI bool `env:"HTTP_SERVE_WEB_UI,default=True"` - SecretKey string `env:"HTTP_SECRET_KEY"` + SecretKey []byte `env:"HTTP_SECRET_KEY"` // Fiber Specific BodyLimit int `env:"HTTP_BODY_LIMIT,default=1024"` ReadTimeout time.Duration `env:"HTTP_READ_TIMEOUT,default=10s"` @@ -82,13 +82,13 @@ type Config struct { // SetDefaults sets the default values for the configuration func (c *HttpConfig) SetDefaults(logger *logrus.Logger) { // Set a random secret key if not set - if c.SecretKey == "" { + if len(c.SecretKey) == 0 { logger.Warn("SHIORI_HTTP_SECRET_KEY is not set, using random value. This means that all sessions will be invalidated on server restart.") randomUUID, err := uuid.NewV4() if err != nil { logger.WithError(err).Fatal("couldn't generate a random UUID") } - c.SecretKey = randomUUID.String() + c.SecretKey = []byte(randomUUID.String()) } } diff --git a/internal/dependencies/dependencies.go b/internal/dependencies/dependencies.go index b51152e40..644607515 100644 --- a/internal/dependencies/dependencies.go +++ b/internal/dependencies/dependencies.go @@ -26,5 +26,6 @@ func NewDependencies(log *logrus.Logger, db database.DB, cfg *config.Config) *De Log: log, Config: cfg, Database: db, + Domains: &Domains{}, } } diff --git a/internal/testutil/shiori.go b/internal/testutil/shiori.go index 48d88cf77..c13d947d4 100644 --- a/internal/testutil/shiori.go +++ b/internal/testutil/shiori.go @@ -19,7 +19,7 @@ func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logg require.NoError(t, err) cfg := config.ParseServerConfiguration(ctx, logger) - cfg.Http.SecretKey = "test" + cfg.Http.SecretKey = []byte("test") tempDir, err := os.MkdirTemp("", "") require.NoError(t, err) From 336528dfe1b0c65b79a28427d3443963e257368c Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 09:39:43 +0100 Subject: [PATCH 09/20] redirect to login on ui errors --- internal/http/response/shortcuts.go | 5 +++++ internal/http/routes/bookmark.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/http/response/shortcuts.go b/internal/http/response/shortcuts.go index 23488fd13..1b3541032 100644 --- a/internal/http/response/shortcuts.go +++ b/internal/http/response/shortcuts.go @@ -32,3 +32,8 @@ func SendErrorWithParams(ctx *gin.Context, statusCode int, data interface{}, err func SendInternalServerError(ctx *gin.Context) { SendError(ctx, http.StatusInternalServerError, internalServerErrorMessage) } + +// SendNotFound directly sends a not found response +func RedirectToLogin(ctx *gin.Context, dst string) { + ctx.Redirect(http.StatusFound, "/login?dst="+dst) +} diff --git a/internal/http/routes/bookmark.go b/internal/http/routes/bookmark.go index 4e7d185d3..9b1a544cd 100644 --- a/internal/http/routes/bookmark.go +++ b/internal/http/routes/bookmark.go @@ -60,7 +60,7 @@ func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, er } if bookmark.Public != 1 && !c.UserIsLogged() { - response.SendError(c.Context, http.StatusForbidden, nil) + response.RedirectToLogin(c.Context, c.Request.URL.String()) return nil, err } From bb64c1e80182ae42e95a3601ac3cdfc4bf92165e Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 09:40:10 +0100 Subject: [PATCH 10/20] fixed archive folder with storage domain --- internal/domains/bookmarks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/domains/bookmarks.go b/internal/domains/bookmarks.go index c18529850..94584715e 100644 --- a/internal/domains/bookmarks.go +++ b/internal/domains/bookmarks.go @@ -26,7 +26,7 @@ func (d *BookmarksDomain) HasEbook(b *model.BookmarkDTO) bool { } func (d *BookmarksDomain) HasArchive(b *model.BookmarkDTO) bool { - archivePath := filepath.Join(d.deps.Config.Storage.DataDir, "archive", strconv.Itoa(b.ID)) + archivePath := filepath.Join("archive", strconv.Itoa(b.ID)) return d.deps.Domains.Storage.FileExists(archivePath) } From 0e971a766871325d5f2f0517c814e8f3e2a7b59e Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 09:49:01 +0100 Subject: [PATCH 11/20] webroot documentation --- docs/Configuration.md | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index f27db7719..942fba5fb 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1,35 +1,53 @@ -Content ---- +# Configuration -- [Content](#content) - [Data Directory](#data-directory) +- [Webroot](#webroot) + - [Nginx](#nginx) - [Database](#database) - - [MySQL](#mysql) - - [PostgreSQL](#postgresql) + - [MySQL](#mysql) + - [PostgreSQL](#postgresql) -Data Directory ---- +## Data Directory Shiori is designed to work out of the box, but you can change where it stores your bookmarks if you need to. By default, Shiori saves your bookmarks in one of the following directories: -| Platform | Directory | -|----------|--------------------------------------------------------------| +| Platform | Directory | +| -------- | ------------------------------------------------------------ | | Linux | `${XDG_DATA_HOME}/shiori` (default: `~/.local/share/shiori`) | -| macOS | `~/Library/Application Support/shiori` | +| macOS | `~/Library/Application Support/shiori` | | Windows | `%LOCALAPPDATA%/shiori` | If you pass the flag `--portable` to Shiori, your data will be stored in the `shiori-data` subdirectory alongside the shiori executable. To specify a custom path, set the `SHIORI_DIR` environment variable. -Database ---- +## Webroot + +If you want to serve Shiori behind a reverse proxy, you can set the `SHIORI_WEBROOT` environment variable to the path where Shiori is served, e.g. `/shiori`. + +Keep in mind this configuration wont make Shiori accessible from `/shiori` path so you need to setup your reverse proxy accordingly so it can strip the webroot path. + +We provide some examples for popular reverse proxies below. Please follow your reverse proxy documentation in order to setup it properly. + +### Nginx + +Fox nginx, you can use the following configuration as a example. The important part **is the trailing slash in `proxy_pass` directive**: + +```nginx +location /shiori { + proxy_pass http://localhost:8080/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; +} +``` + +## Database Shiori uses an SQLite3 database stored in the above data directory by default. If you prefer, you can also use MySQL or PostgreSQL database by setting it in environment variables. From f32035b73b24edbb6aa70962537199d16dde0905 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 10:39:46 +0100 Subject: [PATCH 12/20] some bookmark route tests --- internal/http/routes/bookmark_test.go | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 internal/http/routes/bookmark_test.go diff --git a/internal/http/routes/bookmark_test.go b/internal/http/routes/bookmark_test.go new file mode 100644 index 000000000..406b11aae --- /dev/null +++ b/internal/http/routes/bookmark_test.go @@ -0,0 +1,102 @@ +package routes + +import ( + "context" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/gin-gonic/gin" + sctx "github.com/go-shiori/shiori/internal/http/context" + "github.com/go-shiori/shiori/internal/http/templates" + "github.com/go-shiori/shiori/internal/model" + "github.com/go-shiori/shiori/internal/testutil" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestBookmarkRoutesGetBookmark(t *testing.T) { + logger := logrus.New() + + _, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger) + + g := gin.Default() + templates.SetupTemplates(g) + w := httptest.NewRecorder() + + // Create a private and a public bookmark to use in tests + publicBookmark := testutil.GetValidBookmark() + publicBookmark.Public = 1 + bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, []model.BookmarkDTO{ + *testutil.GetValidBookmark(), + *publicBookmark, + }...) + + require.NoError(t, err) + + router := NewBookmarkRoutes(logger, deps) + + t.Run("bookmark ID is not present", func(t *testing.T) { + gctx := gin.CreateTestContextOnly(w, g) + c := sctx.NewContextFromGin(gctx) + _, err := router.getBookmark(c) + require.Error(t, err) + require.Equal(t, http.StatusBadRequest, c.Writer.Status()) + }) + + t.Run("bookmark ID is not parsable number", func(t *testing.T) { + gctx := gin.CreateTestContextOnly(w, g) + c := sctx.NewContextFromGin(gctx) + c.Params = append(c.Params, gin.Param{Key: "id", Value: "not a number"}) + _, err := router.getBookmark(c) + require.Error(t, err) + require.Equal(t, http.StatusInternalServerError, c.Writer.Status()) + }) + + t.Run("bookmark ID does not exist", func(t *testing.T) { + gctx := gin.CreateTestContextOnly(w, g) + c := sctx.NewContextFromGin(gctx) + c.Params = append(c.Params, gin.Param{Key: "id", Value: "99"}) + bookmark, err := router.getBookmark(c) + require.Equal(t, http.StatusNotFound, c.Writer.Status()) + require.Nil(t, bookmark) + require.Error(t, err) + }) + + t.Run("bookmark ID exists but user is not logged in", func(t *testing.T) { + gctx := gin.CreateTestContextOnly(w, g) + c := sctx.NewContextFromGin(gctx) + c.Request = httptest.NewRequest(http.MethodGet, "/bookmark/1", nil) + c.Params = append(c.Params, gin.Param{Key: "id", Value: "1"}) + bookmark, err := router.getBookmark(c) + require.Equal(t, http.StatusFound, c.Writer.Status()) + require.Nil(t, bookmark) + require.Error(t, err) + }) + + t.Run("bookmark ID exists and its public and user is not logged in", func(t *testing.T) { + gctx := gin.CreateTestContextOnly(w, g) + c := sctx.NewContextFromGin(gctx) + c.Request = httptest.NewRequest(http.MethodGet, "/bookmark/"+strconv.Itoa(bookmarks[0].ID), nil) + c.Params = append(c.Params, gin.Param{Key: "id", Value: strconv.Itoa(bookmarks[1].ID)}) + bookmark, err := router.getBookmark(c) + require.Equal(t, http.StatusOK, c.Writer.Status()) + require.NotNil(t, bookmark) + require.NoError(t, err) + }) + + t.Run("bookmark ID exists and user is logged in", func(t *testing.T) { + g := gin.Default() + templates.SetupTemplates(g) + gctx := gin.CreateTestContextOnly(w, g) + c := sctx.NewContextFromGin(gctx) + c.Set(model.ContextAccountKey, &model.Account{}) + c.Request = httptest.NewRequest(http.MethodGet, "/bookmark/"+strconv.Itoa(bookmarks[0].ID), nil) + c.Params = append(c.Params, gin.Param{Key: "id", Value: strconv.Itoa(bookmarks[0].ID)}) + bookmark, err := router.getBookmark(c) + require.Equal(t, http.StatusOK, c.Writer.Status()) + require.NotNil(t, bookmark) + require.NoError(t, err) + }) +} From d64b2b41a1417d1a5e25a7f3cca7a58067925503 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 10:39:58 +0100 Subject: [PATCH 13/20] fixed error in bookmark domain for non existant bookmarks --- internal/domains/bookmarks.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/domains/bookmarks.go b/internal/domains/bookmarks.go index 94584715e..0e8341a50 100644 --- a/internal/domains/bookmarks.go +++ b/internal/domains/bookmarks.go @@ -39,11 +39,15 @@ func (d *BookmarksDomain) HasThumbnail(b *model.BookmarkDTO) bool { } func (d *BookmarksDomain) GetBookmark(ctx context.Context, id model.DBID) (*model.BookmarkDTO, error) { - bookmark, _, err := d.deps.Database.GetBookmark(ctx, int(id), "") + bookmark, exists, err := d.deps.Database.GetBookmark(ctx, int(id), "") if err != nil { return nil, fmt.Errorf("failed to get bookmark: %w", err) } + if !exists { + return nil, model.ErrBookmarkNotFound + } + // Check if it has ebook and archive. bookmark.HasEbook = d.HasEbook(&bookmark) bookmark.HasArchive = d.HasArchive(&bookmark) From 0f9eb6c1f05a46ea68df20aebcd193e107df8814 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 10:40:10 +0100 Subject: [PATCH 14/20] centralice errors --- internal/http/routes/bookmark.go | 7 ++++--- internal/model/errors.go | 9 +++++++++ internal/testutil/shiori.go | 13 +++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 internal/model/errors.go diff --git a/internal/http/routes/bookmark.go b/internal/http/routes/bookmark.go index 9b1a544cd..1ccdee02d 100644 --- a/internal/http/routes/bookmark.go +++ b/internal/http/routes/bookmark.go @@ -39,6 +39,7 @@ func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, er bookmarkIDParam, present := c.Params.Get("id") if !present { response.SendError(c.Context, 400, "Invalid bookmark ID") + return nil, model.ErrBookmarkInvalidID } bookmarkID, err := strconv.Atoi(bookmarkIDParam) @@ -50,18 +51,18 @@ func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, er if bookmarkID == 0 { response.SendError(c.Context, 404, nil) - return nil, err + return nil, model.ErrBookmarkNotFound } bookmark, err := r.deps.Domains.Bookmarks.GetBookmark(c.Context, model.DBID(bookmarkID)) if err != nil { response.SendError(c.Context, 404, nil) - return nil, err + return nil, model.ErrBookmarkNotFound } if bookmark.Public != 1 && !c.UserIsLogged() { response.RedirectToLogin(c.Context, c.Request.URL.String()) - return nil, err + return nil, model.ErrUnauthorized } return bookmark, nil diff --git a/internal/model/errors.go b/internal/model/errors.go new file mode 100644 index 000000000..75c4b9111 --- /dev/null +++ b/internal/model/errors.go @@ -0,0 +1,9 @@ +package model + +import "errors" + +var ( + ErrBookmarkNotFound = errors.New("bookmark not found") + ErrBookmarkInvalidID = errors.New("invalid bookmark ID") + ErrUnauthorized = errors.New("unauthorized user") +) diff --git a/internal/testutil/shiori.go b/internal/testutil/shiori.go index c13d947d4..9fe96d386 100644 --- a/internal/testutil/shiori.go +++ b/internal/testutil/shiori.go @@ -9,7 +9,8 @@ import ( "github.com/go-shiori/shiori/internal/database" "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/domains" - "github.com/psanford/memfs" + "github.com/go-shiori/shiori/internal/model" + "github.com/gofrs/uuid/v5" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -35,7 +36,15 @@ func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logg deps.Domains.Auth = domains.NewAccountsDomain(deps) deps.Domains.Archiver = domains.NewArchiverDomain(deps) deps.Domains.Bookmarks = domains.NewBookmarksDomain(deps) - deps.Domains.Storage = domains.NewStorageDomain(deps, memfs.New()) + deps.Domains.Storage = domains.NewStorageDomain(deps, os.DirFS(cfg.Storage.DataDir)) return cfg, deps } + +func GetValidBookmark() *model.BookmarkDTO { + uuidV4, _ := uuid.NewV4() + return &model.BookmarkDTO{ + URL: "https://github.com/go-shiori/shiori#" + uuidV4.String(), + Title: "Shiori repository", + } +} From 0bcf5d987dc2e24d7f2fe97c604e956ffd0a7d8e Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 10 Dec 2023 13:46:49 +0100 Subject: [PATCH 15/20] add coverage data to unittests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 676d1cee6..8a7c10641 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BASH ?= $(shell command -v bash 2> /dev/null) SHIORI_DIR ?= dev-data # Testing -GO_TEST_FLAGS ?= -v -race -count=1 +GO_TEST_FLAGS ?= -v -race -count=1 -covermode=atomic -coverprofile=coverage.out GOTESTFMT_FLAGS ?= # Build From f1a13053ca70d069da4712bf71715bffe1c7876f Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Sun, 17 Dec 2023 13:58:08 +0100 Subject: [PATCH 16/20] added tests, refactor storage to use afero --- go.mod | 10 +- go.sum | 21 ++-- internal/cmd/add.go | 2 +- internal/cmd/root.go | 3 +- internal/cmd/update.go | 2 +- internal/core/ebook.go | 15 ++- internal/core/ebook_test.go | 104 ++++++++++--------- internal/core/processing.go | 35 +++---- internal/core/processing_test.go | 73 +++++--------- internal/domains/archiver.go | 12 +-- internal/domains/bookmarks.go | 90 +---------------- internal/domains/bookmarks_test.go | 16 +-- internal/domains/storage.go | 46 ++++++++- internal/domains/storage_test.go | 8 +- internal/domains/utils.go | 46 --------- internal/http/response/file.go | 27 ++++- internal/http/response/shortcuts.go | 4 + internal/http/routes/api/v1/bookmarks.go | 16 +-- internal/http/routes/bookmark.go | 72 ++++---------- internal/http/routes/bookmark_test.go | 121 +++++++++++++++++++++++ internal/model/bookmark.go | 24 ++++- internal/model/domains.go | 7 +- internal/testutil/shiori.go | 5 +- internal/webserver/handler-api-ext.go | 2 +- internal/webserver/handler-api.go | 9 +- 25 files changed, 392 insertions(+), 378 deletions(-) delete mode 100644 internal/domains/utils.go diff --git a/go.mod b/go.mod index 1409c5eb4..24711d0bb 100644 --- a/go.mod +++ b/go.mod @@ -23,10 +23,10 @@ require ( github.com/muesli/go-app-paths v0.2.2 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e github.com/sethvargo/go-envconfig v0.9.0 github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 github.com/swaggo/files v1.0.1 @@ -34,10 +34,10 @@ require ( github.com/swaggo/swag v1.16.2 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f go.uber.org/mock v0.3.0 - golang.org/x/crypto v0.15.0 + golang.org/x/crypto v0.16.0 golang.org/x/image v0.14.0 - golang.org/x/net v0.18.0 - golang.org/x/term v0.14.0 + golang.org/x/net v0.19.0 + golang.org/x/term v0.15.0 modernc.org/sqlite v1.27.0 ) @@ -89,7 +89,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/arch v0.6.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.15.0 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 0cb12a3cf..ba58254b1 100644 --- a/go.sum +++ b/go.sum @@ -112,7 +112,6 @@ github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8 github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -206,8 +205,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e h1:51xcRlSMBU5rhM9KahnJGfEsBPVPz3182TgFRowA8yY= -github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -225,6 +222,8 @@ github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -280,8 +279,8 @@ golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= @@ -297,8 +296,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -320,14 +319,14 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/internal/cmd/add.go b/internal/cmd/add.go index e77b3787f..f099bcdcf 100644 --- a/internal/cmd/add.go +++ b/internal/cmd/add.go @@ -101,7 +101,7 @@ func addHandler(cmd *cobra.Command, args []string) { KeepExcerpt: excerpt != "", } - book, isFatalErr, err = core.ProcessBookmark(request) + book, isFatalErr, err = core.ProcessBookmark(deps, request) content.Close() if err != nil { diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 113074e4c..8dd5ee8e4 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -12,6 +12,7 @@ import ( "github.com/go-shiori/shiori/internal/domains" "github.com/go-shiori/shiori/internal/model" "github.com/sirupsen/logrus" + "github.com/spf13/afero" "github.com/spf13/cobra" "golang.org/x/net/context" ) @@ -101,7 +102,7 @@ func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *depen dependencies.Domains.Auth = domains.NewAccountsDomain(dependencies) dependencies.Domains.Archiver = domains.NewArchiverDomain(dependencies) dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(dependencies) - dependencies.Domains.Storage = domains.NewStorageDomain(dependencies, os.DirFS(cfg.Storage.DataDir)) + dependencies.Domains.Storage = domains.NewStorageDomain(dependencies, afero.NewBasePathFs(afero.NewOsFs(), cfg.Storage.DataDir)) // Workaround: Get accounts to make sure at least one is present in the database. // If there's no accounts in the database, create the shiori/gopher account the legacy api diff --git a/internal/cmd/update.go b/internal/cmd/update.go index e0f8938bf..918ea2a1a 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -175,7 +175,7 @@ func updateHandler(cmd *cobra.Command, args []string) { LogArchival: logArchival, } - book, _, err = core.ProcessBookmark(request) + book, _, err = core.ProcessBookmark(deps, request) content.Close() if err != nil { diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 360cb0982..40cbef4aa 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -1,13 +1,13 @@ package core import ( - "fmt" "os" fp "path/filepath" "strconv" "strings" epub "github.com/go-shiori/go-epub" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/pkg/errors" ) @@ -15,8 +15,7 @@ import ( // GenerateEbook receives a `ProcessRequest` and generates an ebook file in the destination path specified. // The destination path `dstPath` should include file name with ".epub" extension // The bookmark model will be used to update the UI based on whether this function is successful or not. -func GenerateEbook(req ProcessRequest, dstPath string) (book model.BookmarkDTO, err error) { - +func GenerateEbook(deps *dependencies.Dependencies, req ProcessRequest, dstPath string) (book model.BookmarkDTO, err error) { book = req.Bookmark // Make sure bookmark ID is defined @@ -27,14 +26,14 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.BookmarkDTO, // Get current state of bookmark cheak archive and thumb strID := strconv.Itoa(book.ID) - imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID)) - archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) + bookmarkThumbnailPath := model.GetThumbnailPath(&book) + bookmarkArchivePath := model.GetArchivePath(&book) - if _, err := os.Stat(imagePath); err == nil { + if deps.Domains.Storage.FileExists(bookmarkThumbnailPath) { book.ImageURL = fp.Join("/", "bookmark", strID, "thumb") } - if _, err := os.Stat(archivePath); err == nil { + if deps.Domains.Storage.FileExists(bookmarkArchivePath) { book.HasArchive = true } @@ -77,7 +76,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.BookmarkDTO, defer tmpFile.Close() // If everything go well we move ebook to dstPath - err = MoveFileToDestination(dstPath, tmpFile) + err = MoveFileToDestination(deps.Domains.Storage.FS(), dstPath, tmpFile) if err != nil { return book, errors.Wrap(err, "failed move ebook to destination") } diff --git a/internal/core/ebook_test.go b/internal/core/ebook_test.go index 9a3e1bebb..e2f0ce492 100644 --- a/internal/core/ebook_test.go +++ b/internal/core/ebook_test.go @@ -1,23 +1,32 @@ package core_test import ( - "fmt" + "context" "os" fp "path/filepath" "testing" "github.com/go-shiori/shiori/internal/core" + "github.com/go-shiori/shiori/internal/domains" "github.com/go-shiori/shiori/internal/model" + "github.com/go-shiori/shiori/internal/testutil" + "github.com/sirupsen/logrus" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" ) func TestGenerateEbook(t *testing.T) { + logger := logrus.New() + _, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger) + t.Run("Successful ebook generate", func(t *testing.T) { t.Run("valid bookmarkId that return HasEbook true", func(t *testing.T) { // test cae tempDir := t.TempDir() dstDir := t.TempDir() + deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir)) + mockRequest := core.ProcessRequest{ Bookmark: model.BookmarkDTO{ ID: 1, @@ -29,73 +38,65 @@ func TestGenerateEbook(t *testing.T) { ContentType: "text/html", } - bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) + bookmark, err := core.GenerateEbook(deps, mockRequest, fp.Join(tempDir, "1")) assert.True(t, bookmark.HasEbook) assert.NoError(t, err) }) t.Run("ebook generate with valid BookmarkID EbookExist ImagePathExist ReturnWithHasEbookTrue", func(t *testing.T) { - tempDir := t.TempDir() dstDir := t.TempDir() + deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir)) + + bookmark := model.BookmarkDTO{ + ID: 2, + HasEbook: false, + } mockRequest := core.ProcessRequest{ - Bookmark: model.BookmarkDTO{ - ID: 1, - HasEbook: false, - }, + Bookmark: bookmark, DataDir: dstDir, ContentType: "text/html", } - // Create the image directory - imageDir := fp.Join(mockRequest.DataDir, "thumb") - err := os.MkdirAll(imageDir, os.ModePerm) - if err != nil { - t.Fatal(err) - } - // Create the image file - imagePath := fp.Join(mockRequest.DataDir, "thumb", fmt.Sprintf("%d", mockRequest.Bookmark.ID)) - file, err := os.Create(imagePath) + // Create the thumbnail file + imagePath := model.GetThumbnailPath(&bookmark) + deps.Domains.Storage.FS().MkdirAll(fp.Dir(imagePath), os.ModePerm) + file, err := deps.Domains.Storage.FS().Create(imagePath) if err != nil { t.Fatal(err) } defer file.Close() - bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) - expectedimagePath := "/bookmark/1/thumb" - if expectedimagePath != bookmark.ImageURL { - t.Errorf("Expected imageURL %s, but got %s", bookmark.ImageURL, expectedimagePath) - } - assert.True(t, bookmark.HasEbook) + bookmark, err = core.GenerateEbook(deps, mockRequest, model.GetEbookPath(&bookmark)) + expectedImagePath := "/bookmark/2/thumb" assert.NoError(t, err) + assert.True(t, bookmark.HasEbook) + assert.Equalf(t, expectedImagePath, bookmark.ImageURL, "Expected imageURL %s, but got %s", expectedImagePath, bookmark.ImageURL) }) - t.Run("generate ebook valid BookmarkID EbookExist Returnh HasArchive True", func(t *testing.T) { - + t.Run("generate ebook valid BookmarkID EbookExist ReturnHasArchiveTrue", func(t *testing.T) { tempDir := t.TempDir() dstDir := t.TempDir() + deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir)) + + bookmark := model.BookmarkDTO{ + ID: 3, + HasEbook: false, + } mockRequest := core.ProcessRequest{ - Bookmark: model.BookmarkDTO{ - ID: 1, - HasEbook: false, - }, + Bookmark: bookmark, DataDir: dstDir, ContentType: "text/html", } - // Create the archive directory - archiveDir := fp.Join(mockRequest.DataDir, "archive") - err := os.MkdirAll(archiveDir, os.ModePerm) - if err != nil { - t.Fatal(err) - } // Create the archive file - archivePath := fp.Join(mockRequest.DataDir, "archive", fmt.Sprintf("%d", mockRequest.Bookmark.ID)) - file, err := os.Create(archivePath) + archivePath := model.GetArchivePath(&bookmark) + deps.Domains.Storage.FS().MkdirAll(fp.Dir(archivePath), os.ModePerm) + file, err := deps.Domains.Storage.FS().Create(archivePath) if err != nil { t.Fatal(err) } defer file.Close() - bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) + bookmark, err = core.GenerateEbook(deps, mockRequest, fp.Join(tempDir, "1")) assert.True(t, bookmark.HasArchive) assert.NoError(t, err) }) @@ -112,7 +113,7 @@ func TestGenerateEbook(t *testing.T) { ContentType: "text/html", } - bookmark, err := core.GenerateEbook(mockRequest, tempDir) + bookmark, err := core.GenerateEbook(deps, mockRequest, tempDir) assert.Equal(t, model.BookmarkDTO{ ID: 0, @@ -121,32 +122,29 @@ func TestGenerateEbook(t *testing.T) { assert.Error(t, err) }) t.Run("ebook exist return HasEbook true", func(t *testing.T) { - tempDir := t.TempDir() dstDir := t.TempDir() + deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), dstDir)) + + bookmark := model.BookmarkDTO{ + ID: 1, + HasEbook: false, + } mockRequest := core.ProcessRequest{ - Bookmark: model.BookmarkDTO{ - ID: 1, - HasEbook: false, - }, + Bookmark: bookmark, DataDir: dstDir, ContentType: "text/html", } - // Create the ebook directory - ebookDir := fp.Join(mockRequest.DataDir, "ebook") - err := os.MkdirAll(ebookDir, os.ModePerm) - if err != nil { - t.Fatal(err) - } // Create the ebook file - ebookfile := fp.Join(mockRequest.DataDir, "ebook", fmt.Sprintf("%d.epub", mockRequest.Bookmark.ID)) - file, err := os.Create(ebookfile) + ebookFilePath := model.GetEbookPath(&bookmark) + deps.Domains.Storage.FS().MkdirAll(fp.Dir(ebookFilePath), os.ModePerm) + file, err := deps.Domains.Storage.FS().Create(ebookFilePath) if err != nil { t.Fatal(err) } defer file.Close() - bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) + bookmark, err = core.GenerateEbook(deps, mockRequest, ebookFilePath) assert.True(t, bookmark.HasEbook) assert.NoError(t, err) @@ -163,7 +161,7 @@ func TestGenerateEbook(t *testing.T) { ContentType: "application/pdf", } - bookmark, err := core.GenerateEbook(mockRequest, tempDir) + bookmark, err := core.GenerateEbook(deps, mockRequest, tempDir) assert.False(t, bookmark.HasEbook) assert.Error(t, err) diff --git a/internal/core/processing.go b/internal/core/processing.go index 4fb8c6d9c..c97026c27 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -18,9 +18,11 @@ import ( "github.com/disintegration/imaging" "github.com/go-shiori/go-readability" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/go-shiori/warc" "github.com/pkg/errors" + "github.com/spf13/afero" _ "golang.org/x/image/webp" // Add support for png @@ -42,7 +44,7 @@ var ErrNoSupportedImageType = errors.New("unsupported image type") // ProcessBookmark process the bookmark and archive it if needed. // Return three values, is error fatal, and error value. -func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr bool, err error) { +func ProcessBookmark(deps *dependencies.Dependencies, req ProcessRequest) (book model.BookmarkDTO, isFatalErr bool, err error) { book = req.Bookmark contentType := req.ContentType @@ -70,7 +72,7 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo // If this is HTML, parse for readable content strID := strconv.Itoa(book.ID) - imgPath := fp.Join(req.DataDir, "thumb", strID) + imgPath := model.GetThumbnailPath(&book) var imageURLs []string if strings.Contains(contentType, "text/html") { isReadable := readability.Check(readabilityCheckInput) @@ -107,7 +109,7 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo if article.Image != "" { imageURLs = append(imageURLs, article.Image) } else { - os.Remove(imgPath) + deps.Domains.Storage.FS().Remove(imgPath) } if article.Favicon != "" { @@ -123,11 +125,11 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo // Save article image to local disk for i, imageURL := range imageURLs { - err = DownloadBookImage(imageURL, imgPath) + err = DownloadBookImage(deps.Domains.Storage.FS(), imageURL, imgPath) if err != nil && errors.Is(err, ErrNoSupportedImageType) { log.Printf("%s: %s", err, imageURL) if i == len(imageURLs)-1 { - os.Remove(imgPath) + deps.Domains.Storage.FS().Remove(imgPath) } } if err != nil { @@ -142,13 +144,13 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo // If needed, create ebook as well if book.CreateEbook { - ebookPath := fp.Join(req.DataDir, "ebook", strID+".epub") + ebookPath := model.GetEbookPath(&book) req.Bookmark = book if strings.Contains(contentType, "application/pdf") { return book, false, errors.Wrap(err, "can't create ebook from pdf") } else { - _, err = GenerateEbook(req, ebookPath) + _, err = GenerateEbook(deps, req, ebookPath) if err != nil { return book, true, errors.Wrap(err, "failed to create ebook") } @@ -162,7 +164,7 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo if err != nil { return book, false, fmt.Errorf("failed to create temp archive: %v", err) } - defer os.Remove(tmpFile.Name()) + defer deps.Domains.Storage.FS().Remove(tmpFile.Name()) archivalRequest := warc.ArchivalRequest{ URL: book.URL, @@ -178,10 +180,8 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo return book, false, fmt.Errorf("failed to create archive: %v", err) } - // Prepare destination file. - dstPath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) - - err = MoveFileToDestination(dstPath, tmpFile) + dstPath := model.GetArchivePath(&book) + err = MoveFileToDestination(deps.Domains.Storage.FS(), dstPath, tmpFile) if err != nil { return book, false, fmt.Errorf("failed move archive to destination `: %v", err) } @@ -192,7 +192,7 @@ func ProcessBookmark(req ProcessRequest) (book model.BookmarkDTO, isFatalErr boo return book, false, nil } -func DownloadBookImage(url, dstPath string) error { +func DownloadBookImage(fs afero.Fs, url, dstPath string) error { // Fetch data from URL resp, err := httpClient.Get(url) if err != nil { @@ -264,7 +264,7 @@ func DownloadBookImage(url, dstPath string) error { return fmt.Errorf("failed to save image %s: %v", url, err) } - err = MoveFileToDestination(dstPath, tmpFile) + err = MoveFileToDestination(fs, dstPath, tmpFile) if err != nil { return err } @@ -273,14 +273,15 @@ func DownloadBookImage(url, dstPath string) error { } // dstPath requires the filename -func MoveFileToDestination(dstPath string, tmpFile *os.File) error { +// TODO: move to storage domain +func MoveFileToDestination(fs afero.Fs, dstPath string, tmpFile *os.File) error { // Prepare destination file. - err := os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) + err := fs.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) if err != nil { return fmt.Errorf("failed to create destination dir: %v", err) } - dstFile, err := os.Create(dstPath) + dstFile, err := fs.Create(dstPath) if err != nil { return fmt.Errorf("failed to create destination file: %v", err) } diff --git a/internal/core/processing_test.go b/internal/core/processing_test.go index a46ca8425..531eb1550 100644 --- a/internal/core/processing_test.go +++ b/internal/core/processing_test.go @@ -2,6 +2,7 @@ package core_test import ( "bytes" + "context" "net/http" "net/http/httptest" "os" @@ -10,43 +11,15 @@ import ( "github.com/go-shiori/shiori/internal/core" "github.com/go-shiori/shiori/internal/model" + "github.com/go-shiori/shiori/internal/testutil" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) -func TestMoveFileToDestination(t *testing.T) { - t.Run("create fails", func(t *testing.T) { - t.Run("directory create fails", func(t *testing.T) { - // test if create dir fails - tmpFile, err := os.CreateTemp("", "image") - - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - - err = core.MoveFileToDestination("/destination/test", tmpFile) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to create destination dir") - }) - t.Run("file create fails", func(t *testing.T) { - // if create file failed - tmpFile, err := os.CreateTemp("", "image") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - - // Create a destination directory - dstDir := t.TempDir() - assert.NoError(t, err) - defer os.Remove(dstDir) - - // Set destination path to an invalid file name to force os.Create to fail - dstPath := fp.Join(dstDir, "\000invalid\000") - - err = core.MoveFileToDestination(dstPath, tmpFile) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to create destination file") - }) - }) -} func TestDownloadBookImage(t *testing.T) { + logger := logrus.New() + _, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger) + t.Run("Download Images", func(t *testing.T) { t.Run("fails", func(t *testing.T) { // images is too small with unsupported format with a valid URL @@ -56,13 +29,13 @@ func TestDownloadBookImage(t *testing.T) { defer os.Remove(dstPath) // Act - err := core.DownloadBookImage(imageURL, dstPath) + err := core.DownloadBookImage(deps.Domains.Storage.FS(), imageURL, dstPath) // Assert assert.EqualError(t, err, "unsupported image type") - assert.NoFileExists(t, dstPath) + assert.False(t, deps.Domains.Storage.FileExists(dstPath)) }) - t.Run("sucssesful downlosd image", func(t *testing.T) { + t.Run("successful download image", func(t *testing.T) { // Arrange imageURL := "https://raw.githubusercontent.com/go-shiori/shiori/master/docs/readme/cover.png" tempDir := t.TempDir() @@ -70,13 +43,13 @@ func TestDownloadBookImage(t *testing.T) { defer os.Remove(dstPath) // Act - err := core.DownloadBookImage(imageURL, dstPath) + err := core.DownloadBookImage(deps.Domains.Storage.FS(), imageURL, dstPath) // Assert assert.NoError(t, err) - assert.FileExists(t, dstPath) + assert.True(t, deps.Domains.Storage.FileExists(dstPath)) }) - t.Run("sucssesful downlosd medium size image", func(t *testing.T) { + t.Run("successful download medium size image", func(t *testing.T) { // create a file server handler for the 'testdata' directory fs := http.FileServer(http.Dir("../../testdata/")) @@ -91,17 +64,19 @@ func TestDownloadBookImage(t *testing.T) { defer os.Remove(dstPath) // Act - err := core.DownloadBookImage(imageURL, dstPath) + err := core.DownloadBookImage(deps.Domains.Storage.FS(), imageURL, dstPath) // Assert assert.NoError(t, err) - assert.FileExists(t, dstPath) - + assert.True(t, deps.Domains.Storage.FileExists(dstPath)) }) }) } func TestProcessBookmark(t *testing.T) { + logger := logrus.New() + _, deps := testutil.GetTestConfigurationAndDependencies(t, context.TODO(), logger) + t.Run("ProcessRequest with sucssesful result", func(t *testing.T) { t.Run("Normal without image", func(t *testing.T) { bookmark := model.BookmarkDTO{ @@ -121,7 +96,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: true, } - expected, _, _ := core.ProcessBookmark(request) + expected, _, _ := core.ProcessBookmark(deps, request) if expected.ID != bookmark.ID { t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) @@ -165,7 +140,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: true, } - expected, _, _ := core.ProcessBookmark(request) + expected, _, _ := core.ProcessBookmark(deps, request) if expected.ID != bookmark.ID { t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) @@ -215,7 +190,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: true, } - expected, _, _ := core.ProcessBookmark(request) + expected, _, _ := core.ProcessBookmark(deps, request) if expected.ID != bookmark.ID { t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) @@ -248,7 +223,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: true, } - expected, _, _ := core.ProcessBookmark(request) + expected, _, _ := core.ProcessBookmark(deps, request) if expected.ID != bookmark.ID { t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) @@ -281,7 +256,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: false, } - expected, _, _ := core.ProcessBookmark(request) + expected, _, _ := core.ProcessBookmark(deps, request) if expected.ID != bookmark.ID { t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) @@ -316,7 +291,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: true, } - _, isFatal, err := core.ProcessBookmark(request) + _, isFatal, err := core.ProcessBookmark(deps, request) assert.Error(t, err) assert.Contains(t, err.Error(), "bookmark ID is not valid") assert.True(t, isFatal) @@ -341,7 +316,7 @@ func TestProcessBookmark(t *testing.T) { KeepTitle: true, KeepExcerpt: true, } - _, _, err := core.ProcessBookmark(request) + _, _, err := core.ProcessBookmark(deps, request) assert.NoError(t, err) }) }) diff --git a/internal/domains/archiver.go b/internal/domains/archiver.go index e2488d094..358523b54 100644 --- a/internal/domains/archiver.go +++ b/internal/domains/archiver.go @@ -3,7 +3,6 @@ package domains import ( "fmt" "path/filepath" - "strconv" "github.com/go-shiori/shiori/internal/core" "github.com/go-shiori/shiori/internal/dependencies" @@ -28,7 +27,7 @@ func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model ContentType: contentType, } - result, isFatalErr, err := core.ProcessBookmark(processRequest) + result, isFatalErr, err := core.ProcessBookmark(d.deps, processRequest) content.Close() if err != nil && isFatalErr { @@ -39,13 +38,14 @@ func (d *ArchiverDomain) DownloadBookmarkArchive(book model.BookmarkDTO) (*model } func (d *ArchiverDomain) GetBookmarkArchive(book *model.BookmarkDTO) (*warc.Archive, error) { - archivePath := filepath.Join(d.deps.Config.Storage.DataDir, "archive", strconv.Itoa(book.ID)) + archivePath := model.GetArchivePath(book) - if !FileExists(archivePath) { - return nil, fmt.Errorf("archive not found") + if !d.deps.Domains.Storage.FileExists(archivePath) { + return nil, fmt.Errorf("archive for bookmark %d doesn't exist", book.ID) } - return warc.Open(archivePath) + // FIXME: This only works in local filesystem + return warc.Open(filepath.Join(d.deps.Config.Storage.DataDir, archivePath)) } func NewArchiverDomain(deps *dependencies.Dependencies) *ArchiverDomain { diff --git a/internal/domains/bookmarks.go b/internal/domains/bookmarks.go index 0e8341a50..df5d968dd 100644 --- a/internal/domains/bookmarks.go +++ b/internal/domains/bookmarks.go @@ -3,17 +3,9 @@ package domains import ( "context" "fmt" - "html/template" - "net/url" - "path" - "path/filepath" - "strconv" - "strings" - "github.com/PuerkitoBio/goquery" "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" - "github.com/go-shiori/warc" ) type BookmarksDomain struct { @@ -21,21 +13,18 @@ type BookmarksDomain struct { } func (d *BookmarksDomain) HasEbook(b *model.BookmarkDTO) bool { - ebookPath := filepath.Join("ebook", strconv.Itoa(b.ID)+".epub") + ebookPath := model.GetEbookPath(b) return d.deps.Domains.Storage.FileExists(ebookPath) } func (d *BookmarksDomain) HasArchive(b *model.BookmarkDTO) bool { - archivePath := filepath.Join("archive", strconv.Itoa(b.ID)) + archivePath := model.GetArchivePath(b) return d.deps.Domains.Storage.FileExists(archivePath) } -func (d *BookmarksDomain) GetThumbnailPath(b *model.BookmarkDTO) string { - return filepath.Join("thumb", strconv.Itoa(b.ID)) -} - func (d *BookmarksDomain) HasThumbnail(b *model.BookmarkDTO) bool { - return d.deps.Domains.Storage.FileExists(d.GetThumbnailPath(b)) + thumbnailPath := model.GetThumbnailPath(b) + return d.deps.Domains.Storage.FileExists(thumbnailPath) } func (d *BookmarksDomain) GetBookmark(ctx context.Context, id model.DBID) (*model.BookmarkDTO, error) { @@ -55,77 +44,6 @@ func (d *BookmarksDomain) GetBookmark(ctx context.Context, id model.DBID) (*mode return &bookmark, nil } -// GetBookmarkContentsFromArchive gets the HTML contents of a bookmark linking assets to the ones -// archived if the bookmark has an archived version. -func (d *BookmarksDomain) GetBookmarkContentsFromArchive(bookmark *model.BookmarkDTO) (template.HTML, error) { - if !bookmark.HasArchive { - return template.HTML(bookmark.HTML), nil - } - - // Open archive, look in cache first - archivePath := filepath.Join(d.deps.Config.Storage.DataDir, "archive", fmt.Sprintf("%d", bookmark.ID)) - // TODO: Move to archiver domain - // TODO: Use storagedomain to operate with the file - archive, err := warc.Open(archivePath) - if err != nil { - return "", fmt.Errorf("failed to open archive: %w", err) - } - - // Find all image and convert its source to use the archive URL. - createArchivalURL := func(archivalName string) string { - var archivalURL url.URL - archivalURL.Path = path.Join(d.deps.Config.Http.RootPath, "bookmark", fmt.Sprintf("%d", bookmark.ID), "archive", archivalName) - return archivalURL.String() - } - - buffer := strings.NewReader(bookmark.HTML) - doc, err := goquery.NewDocumentFromReader(buffer) - if err != nil { - return "", fmt.Errorf("failed to parse HTML: %w", err) - } - - doc.Find("img, picture, figure, source").Each(func(_ int, node *goquery.Selection) { - // Get the needed attributes - src, _ := node.Attr("src") - strSrcSets, _ := node.Attr("srcset") - - // Convert `src` attributes - if src != "" { - archivalName := getArchiveFileBasename(src) - if archivalName != "" && archive.HasResource(archivalName) { - node.SetAttr("src", createArchivalURL(archivalName)) - } - } - - // Split srcset by comma, then process it like any URLs - srcSets := strings.Split(strSrcSets, ",") - for i, srcSet := range srcSets { - srcSet = strings.TrimSpace(srcSet) - parts := strings.SplitN(srcSet, " ", 2) - if parts[0] == "" { - continue - } - - archivalName := getArchiveFileBasename(parts[0]) - if archivalName != "" && archive.HasResource(archivalName) { - archivalURL := createArchivalURL(archivalName) - srcSets[i] = strings.Replace(srcSets[i], parts[0], archivalURL, 1) - } - } - - if len(srcSets) > 0 { - node.SetAttr("srcset", strings.Join(srcSets, ",")) - } - }) - - bookmark.HTML, err = goquery.OuterHtml(doc.Selection) - if err != nil { - return template.HTML(bookmark.HTML), fmt.Errorf("failed to get HTML: %w", err) - } - - return template.HTML(bookmark.HTML), nil -} - func NewBookmarksDomain(deps *dependencies.Dependencies) *BookmarksDomain { return &BookmarksDomain{ deps: deps, diff --git a/internal/domains/bookmarks_test.go b/internal/domains/bookmarks_test.go index 1523a8667..5b3d724b0 100644 --- a/internal/domains/bookmarks_test.go +++ b/internal/domains/bookmarks_test.go @@ -2,7 +2,6 @@ package domains_test import ( "context" - "path/filepath" "testing" "github.com/go-shiori/shiori/internal/config" @@ -10,14 +9,14 @@ import ( "github.com/go-shiori/shiori/internal/domains" "github.com/go-shiori/shiori/internal/mocks" "github.com/go-shiori/shiori/internal/model" - "github.com/psanford/memfs" "github.com/sirupsen/logrus" + "github.com/spf13/afero" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) func TestBookmarkDomain(t *testing.T) { - fs := memfs.New() + fs := afero.NewMemMapFs() mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -31,12 +30,12 @@ func TestBookmarkDomain(t *testing.T) { deps.Domains.Storage = domains.NewStorageDomain(deps, fs) fs.MkdirAll("thumb", 0755) - fs.WriteFile("thumb/1", []byte("x"), 0644) + fs.Create("thumb/1") fs.MkdirAll("ebook", 0755) - fs.WriteFile("ebook/1.epub", []byte("x"), 0644) + fs.Create("ebook/1.epub") fs.MkdirAll("archive", 0755) // TODO: write a valid archive file - fs.WriteFile("archive/1", []byte("x"), 0644) + fs.Create("archive/1") domain := domains.NewBookmarksDomain(deps) t.Run("HasEbook", func(t *testing.T) { @@ -57,11 +56,6 @@ func TestBookmarkDomain(t *testing.T) { }) }) - t.Run("GetThumbnailPath", func(t *testing.T) { - thumbnailPath := domain.GetThumbnailPath(&model.BookmarkDTO{ID: 1}) - require.Equal(t, filepath.Join("thumb/1"), thumbnailPath) - }) - t.Run("HasThumbnail", func(t *testing.T) { t.Run("Yes", func(t *testing.T) { require.True(t, domain.HasThumbnail(&model.BookmarkDTO{ID: 1})) diff --git a/internal/domains/storage.go b/internal/domains/storage.go index b39e817d1..186d031f3 100644 --- a/internal/domains/storage.go +++ b/internal/domains/storage.go @@ -2,29 +2,65 @@ package domains import ( "io/fs" + "os" + "path/filepath" "github.com/go-shiori/shiori/internal/dependencies" + "github.com/spf13/afero" ) type StorageDomain struct { deps *dependencies.Dependencies - fs fs.FS + fs afero.Fs +} + +// Stat returns the FileInfo structure describing file. +func (d *StorageDomain) Stat(name string) (fs.FileInfo, error) { + return d.fs.Stat(name) +} + +// FS returns the filesystem used by this domain. +func (d *StorageDomain) FS() afero.Fs { + return d.fs } // FileExists checks if a file exists in storage. func (d *StorageDomain) FileExists(name string) bool { - info, err := fs.Stat(d.fs, name) + info, err := d.Stat(name) return err == nil && !info.IsDir() } // DirExists checks if a directory exists in storage. func (d *StorageDomain) DirExists(name string) bool { - info, err := fs.Stat(d.fs, name) - d.deps.Log.Info(info) + info, err := d.Stat(name) return err == nil && info.IsDir() } -func NewStorageDomain(deps *dependencies.Dependencies, fs fs.FS) *StorageDomain { +// Write writes data to a file in storage. +// CAUTION: This function will overwrite existing file. +func (d *StorageDomain) Save(name string, data []byte) error { + // Create directory if not exist + dir := filepath.Dir(name) + if !d.DirExists(dir) { + err := d.fs.MkdirAll(dir, os.ModePerm) + if err != nil { + return err + } + } + + // Create file + file, err := d.fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer file.Close() + + // Write data + _, err = file.Write(data) + return err +} + +func NewStorageDomain(deps *dependencies.Dependencies, fs afero.Fs) *StorageDomain { return &StorageDomain{ deps: deps, fs: fs, diff --git a/internal/domains/storage_test.go b/internal/domains/storage_test.go index ee913faf0..7e6a28038 100644 --- a/internal/domains/storage_test.go +++ b/internal/domains/storage_test.go @@ -7,13 +7,13 @@ import ( "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/domains" - "github.com/psanford/memfs" "github.com/sirupsen/logrus" + "github.com/spf13/afero" "github.com/stretchr/testify/require" ) func TestDirExists(t *testing.T) { - fs := memfs.New() + fs := afero.NewMemMapFs() fs.MkdirAll("foo", 0755) domain := domains.NewStorageDomain( @@ -30,9 +30,9 @@ func TestDirExists(t *testing.T) { } func TestFileExists(t *testing.T) { - fs := memfs.New() + fs := afero.NewMemMapFs() fs.MkdirAll("foo", 0755) - fs.WriteFile("foo/file", []byte("hello world"), 0644) + fs.Create("foo/file") domain := domains.NewStorageDomain( &dependencies.Dependencies{ diff --git a/internal/domains/utils.go b/internal/domains/utils.go deleted file mode 100644 index 1b45a85d8..000000000 --- a/internal/domains/utils.go +++ /dev/null @@ -1,46 +0,0 @@ -package domains - -import ( - "net/url" - "os" - "regexp" - "strings" -) - -var rxRepeatedStrip = regexp.MustCompile(`(?i)-+`) - -func FileExists(path string) bool { - info, err := os.Stat(path) - return err == nil && !info.IsDir() -} - -// getArchiveFileBasename converts an URL into an archival name. -func getArchiveFileBasename(src string) string { - archivalURL := src - - // Some URL have its query or path escaped, e.g. Wikipedia and Dev.to. - // For example, Wikipedia's stylesheet looks like this : - // load.php?lang=en&modules=ext.3d.styles%7Cext.cite.styles%7Cext.uls.interlanguage - // However, when browser download it, it will be registered as unescaped query : - // load.php?lang=en&modules=ext.3d.styles|ext.cite.styles|ext.uls.interlanguage - // So, for archival URL, we need to unescape the query and path first. - tmp, err := url.Parse(src) - if err == nil { - unescapedQuery, _ := url.QueryUnescape(tmp.RawQuery) - if unescapedQuery != "" { - tmp.RawQuery = unescapedQuery - } - - archivalURL = tmp.String() - archivalURL = strings.Replace(archivalURL, tmp.EscapedPath(), tmp.Path, 1) - } - - archivalURL = strings.ReplaceAll(archivalURL, "://", "/") - archivalURL = strings.ReplaceAll(archivalURL, "?", "-") - archivalURL = strings.ReplaceAll(archivalURL, "#", "-") - archivalURL = strings.ReplaceAll(archivalURL, "/", "-") - archivalURL = strings.ReplaceAll(archivalURL, " ", "-") - archivalURL = rxRepeatedStrip.ReplaceAllString(archivalURL, "-") - - return archivalURL -} diff --git a/internal/http/response/file.go b/internal/http/response/file.go index d97aa03a8..d0704a8de 100644 --- a/internal/http/response/file.go +++ b/internal/http/response/file.go @@ -2,22 +2,41 @@ package response import ( "fmt" + "io" "net/http" - "os" "github.com/gin-gonic/gin" + "github.com/go-shiori/shiori/internal/model" ) // SendFile sends file to client with caching header -func SendFile(c *gin.Context, path string) { +func SendFile(c *gin.Context, storageDomain model.StorageDomain, path string) { c.Header("Cache-Control", "public, max-age=86400") - info, err := os.Stat(path) + if !storageDomain.FileExists(path) { + c.AbortWithStatus(http.StatusNotFound) + return + } + + info, err := storageDomain.Stat(path) if err != nil { c.AbortWithStatus(http.StatusInternalServerError) return } c.Header("ETag", fmt.Sprintf("W/%x-%x", info.ModTime().Unix(), info.Size())) - c.File(path) + + // TODO: Find a better way to send the file to the client from the FS, probably making a + // conversion between afero.Fs and http.FileSystem to use c.FileFromFS. + fileContent, err := storageDomain.FS().Open(path) + if err != nil { + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + _, err = io.Copy(c.Writer, fileContent) + if err != nil { + c.AbortWithStatus(http.StatusInternalServerError) + return + } } diff --git a/internal/http/response/shortcuts.go b/internal/http/response/shortcuts.go index 1b3541032..78e0bb0aa 100644 --- a/internal/http/response/shortcuts.go +++ b/internal/http/response/shortcuts.go @@ -37,3 +37,7 @@ func SendInternalServerError(ctx *gin.Context) { func RedirectToLogin(ctx *gin.Context, dst string) { ctx.Redirect(http.StatusFound, "/login?dst="+dst) } + +func NotFound(ctx *gin.Context) { + ctx.AbortWithStatus(http.StatusNotFound) +} diff --git a/internal/http/routes/api/v1/bookmarks.go b/internal/http/routes/api/v1/bookmarks.go index 4b5f9eba9..6d75c1eee 100644 --- a/internal/http/routes/api/v1/bookmarks.go +++ b/internal/http/routes/api/v1/bookmarks.go @@ -33,6 +33,13 @@ func (r *BookmarksAPIRoutes) Setup(g *gin.RouterGroup) model.Routes { return r } +func NewBookmarksPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarksAPIRoutes { + return &BookmarksAPIRoutes{ + logger: logger, + deps: deps, + } +} + type updateCachePayload struct { Ids []int `json:"ids" validate:"required"` KeepMetadata bool `json:"keep_metadata"` @@ -189,13 +196,6 @@ func (r *BookmarksAPIRoutes) deleteHandler(c *gin.Context) { response.Send(c, 200, "Bookmark deleted") } -func NewBookmarksPIRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarksAPIRoutes { - return &BookmarksAPIRoutes{ - logger: logger, - deps: deps, - } -} - // updateCache godoc // // @Summary Update Cache and Ebook on server. @@ -298,7 +298,7 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) { } } - book, _, err = core.ProcessBookmark(request) + book, _, err = core.ProcessBookmark(r.deps, request) content.Close() if err != nil { diff --git a/internal/http/routes/bookmark.go b/internal/http/routes/bookmark.go index 1ccdee02d..ae2cc0905 100644 --- a/internal/http/routes/bookmark.go +++ b/internal/http/routes/bookmark.go @@ -7,15 +7,11 @@ import ( "strconv" "strings" - fp "path/filepath" - "github.com/gin-gonic/gin" - "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/http/context" "github.com/go-shiori/shiori/internal/http/response" "github.com/go-shiori/shiori/internal/model" - ws "github.com/go-shiori/shiori/internal/webserver" "github.com/gofrs/uuid/v5" "github.com/sirupsen/logrus" ) @@ -25,6 +21,13 @@ type BookmarkRoutes struct { deps *dependencies.Dependencies } +func NewBookmarkRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarkRoutes { + return &BookmarkRoutes{ + logger: logger, + deps: deps, + } +} + func (r *BookmarkRoutes) Setup(group *gin.RouterGroup) model.Routes { group.GET("/:id/archive", r.bookmarkArchiveHandler) group.GET("/:id/archive/file/*filepath", r.bookmarkArchiveFileHandler) @@ -38,7 +41,7 @@ func (r *BookmarkRoutes) Setup(group *gin.RouterGroup) model.Routes { func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, error) { bookmarkIDParam, present := c.Params.Get("id") if !present { - response.SendError(c.Context, 400, "Invalid bookmark ID") + response.SendError(c.Context, http.StatusBadRequest, "Invalid bookmark ID") return nil, model.ErrBookmarkInvalidID } @@ -50,13 +53,13 @@ func (r *BookmarkRoutes) getBookmark(c *context.Context) (*model.BookmarkDTO, er } if bookmarkID == 0 { - response.SendError(c.Context, 404, nil) + response.SendError(c.Context, http.StatusNotFound, nil) return nil, model.ErrBookmarkNotFound } bookmark, err := r.deps.Domains.Bookmarks.GetBookmark(c.Context, model.DBID(bookmarkID)) if err != nil { - response.SendError(c.Context, 404, nil) + response.SendError(c.Context, http.StatusNotFound, nil) return nil, model.ErrBookmarkNotFound } @@ -93,7 +96,7 @@ func (r *BookmarkRoutes) bookmarkArchiveHandler(c *gin.Context) { } if !r.deps.Domains.Bookmarks.HasArchive(bookmark) { - response.SendError(c, http.StatusNotFound, nil) + response.NotFound(c) return } @@ -113,7 +116,7 @@ func (r *BookmarkRoutes) bookmarkArchiveFileHandler(c *gin.Context) { } if !r.deps.Domains.Bookmarks.HasArchive(bookmark) { - response.SendError(c, http.StatusNotFound, nil) + response.NotFound(c) return } @@ -129,7 +132,7 @@ func (r *BookmarkRoutes) bookmarkArchiveFileHandler(c *gin.Context) { defer archive.Close() if !archive.HasResource(resourcePath) { - response.SendError(c, http.StatusNotFound, nil) + response.NotFound(c) return } @@ -158,62 +161,29 @@ func (r *BookmarkRoutes) bookmarkThumbnailHandler(c *gin.Context) { } if !r.deps.Domains.Bookmarks.HasThumbnail(bookmark) { - response.SendError(c, http.StatusNotFound, nil) + response.NotFound(c) return } - response.SendFile(c, r.deps.Domains.Bookmarks.GetThumbnailPath(bookmark)) -} - -func NewBookmarkRoutes(logger *logrus.Logger, deps *dependencies.Dependencies) *BookmarkRoutes { - return &BookmarkRoutes{ - logger: logger, - deps: deps, - } + response.SendFile(c, r.deps.Domains.Storage, model.GetThumbnailPath(bookmark)) } func (r *BookmarkRoutes) bookmarkEbookHandler(c *gin.Context) { ctx := context.NewContextFromGin(c) - // Get server config - logger := logrus.New() - cfg := config.ParseServerConfiguration(ctx, logger) - DataDir := cfg.Storage.DataDir - - bookmarkIDParam, present := c.Params.Get("id") - if !present { - response.SendError(c, http.StatusBadRequest, "Invalid bookmark ID") - return - } - - bookmarkID, err := strconv.Atoi(bookmarkIDParam) + bookmark, err := r.getBookmark(ctx) if err != nil { - r.logger.WithError(err).Error("error parsing bookmark ID parameter") - response.SendInternalServerError(c) return } - if bookmarkID == 0 { - response.SendError(c, http.StatusNotFound, nil) - return - } + ebookPath := model.GetEbookPath(bookmark) - bookmark, found, err := r.deps.Database.GetBookmark(c, bookmarkID, "") - if err != nil || !found { + if !r.deps.Domains.Storage.FileExists(ebookPath) { response.SendError(c, http.StatusNotFound, nil) return } - if bookmark.Public != 1 && !ctx.UserIsLogged() { - response.SendError(c, http.StatusUnauthorized, nil) - return - } - - ebookPath := fp.Join(DataDir, "ebook", bookmarkIDParam+".epub") - if !ws.FileExists(ebookPath) { - response.SendError(c, http.StatusNotFound, nil) - return - } - filename := bookmark.Title + ".epub" - c.FileAttachment(ebookPath, filename) + // TODO: Potentially improve this + c.Header("Content-Disposition", `attachment; filename="`+bookmark.Title+`.epub"`) + response.SendFile(c, r.deps.Domains.Storage, model.GetEbookPath(bookmark)) } diff --git a/internal/http/routes/bookmark_test.go b/internal/http/routes/bookmark_test.go index 406b11aae..400dd95b2 100644 --- a/internal/http/routes/bookmark_test.go +++ b/internal/http/routes/bookmark_test.go @@ -100,3 +100,124 @@ func TestBookmarkRoutesGetBookmark(t *testing.T) { require.NoError(t, err) }) } + +func TestBookmarkContentHandler(t *testing.T) { + logger := logrus.New() + + _, deps := testutil.GetTestConfigurationAndDependencies(t, context.Background(), logger) + + bookmark := testutil.GetValidBookmark() + bookmark.HTML = "

Bookmark HTML content

" + boomkarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark) + require.NoError(t, err) + + bookmark = &boomkarks[0] + + t.Run("not logged in", func(t *testing.T) { + g := gin.Default() + router := NewBookmarkRoutes(logger, deps) + router.Setup(g.Group("/")) + w := httptest.NewRecorder() + path := "/" + strconv.Itoa(bookmark.ID) + "/content" + req, _ := http.NewRequest("GET", path, nil) + g.ServeHTTP(w, req) + require.Equal(t, http.StatusFound, w.Code) + require.Equal(t, "/login?dst="+path, w.Header().Get("Location")) + }) + + t.Run("get existing bookmark content", func(t *testing.T) { + g := gin.Default() + templates.SetupTemplates(g) + g.Use(func(c *gin.Context) { + c.Set(model.ContextAccountKey, "test") + }) + router := NewBookmarkRoutes(logger, deps) + router.Setup(g.Group("/")) + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/content", nil) + g.ServeHTTP(w, req) + t.Log(w.Header().Get("Location")) + require.Equal(t, 200, w.Code) + require.Contains(t, w.Body.String(), bookmark.HTML) + }) +} + +func TestBookmarkFileHandlers(t *testing.T) { + logger := logrus.New() + + _, deps := testutil.GetTestConfigurationAndDependencies(t, context.Background(), logger) + + bookmark := testutil.GetValidBookmark() + bookmark.HTML = "

Bookmark HTML content

" + bookmark.HasArchive = true + bookmark.CreateArchive = true + bookmark.CreateEbook = true + bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark) + require.NoError(t, err) + + bookmark, err = deps.Domains.Archiver.DownloadBookmarkArchive(bookmarks[0]) + require.NoError(t, err) + + bookmarks, err = deps.Database.SaveBookmarks(context.TODO(), false, *bookmark) + require.NoError(t, err) + bookmark = &bookmarks[0] + + g := gin.Default() + templates.SetupTemplates(g) + g.Use(func(c *gin.Context) { + c.Set(model.ContextAccountKey, "test") + }) + router := NewBookmarkRoutes(logger, deps) + router.Setup(g.Group("/")) + + t.Run("get existing bookmark archive", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/archive", nil) + g.ServeHTTP(w, req) + require.Contains(t, w.Body.String(), "iframe") + require.Equal(t, 200, w.Code) + }) + + t.Run("get existing bookmark thumbnail", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/thumb", nil) + g.ServeHTTP(w, req) + require.Equal(t, 200, w.Code) + }) + + t.Run("bookmark without archive", func(t *testing.T) { + bookmark := testutil.GetValidBookmark() + bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark) + require.NoError(t, err) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmarks[0].ID)+"/archive", nil) + g.ServeHTTP(w, req) + require.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("get existing bookmark archive file", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmark.ID)+"/archive/file/", nil) + g.ServeHTTP(w, req) + require.Equal(t, 200, w.Code) + }) + + t.Run("bookmark with ebook", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmarks[0].ID)+"/ebook", nil) + g.ServeHTTP(w, req) + require.Equal(t, http.StatusOK, w.Code) + }) + + t.Run("bookmark without ebook", func(t *testing.T) { + bookmark := testutil.GetValidBookmark() + bookmarks, err := deps.Database.SaveBookmarks(context.TODO(), true, *bookmark) + require.NoError(t, err) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/"+strconv.Itoa(bookmarks[0].ID)+"/ebook", nil) + g.ServeHTTP(w, req) + require.Equal(t, http.StatusNotFound, w.Code) + }) +} diff --git a/internal/model/bookmark.go b/internal/model/bookmark.go index e2dcd5c44..a85ec2a01 100644 --- a/internal/model/bookmark.go +++ b/internal/model/bookmark.go @@ -1,5 +1,10 @@ package model +import ( + "path/filepath" + "strconv" +) + // BookmarkDTO is the bookmark object representation in database and the data transfer object // at the same time, pending a refactor to two separate object to represent each role. type BookmarkDTO struct { @@ -17,6 +22,21 @@ type BookmarkDTO struct { Tags []Tag `json:"tags"` HasArchive bool `json:"hasArchive"` HasEbook bool `json:"hasEbook"` - CreateArchive bool `json:"create_archive"` - CreateEbook bool `json:"create_ebook"` + CreateArchive bool `json:"create_archive"` // TODO: migrate outside the DTO + CreateEbook bool `json:"create_ebook"` // TODO: migrate outside the DTO +} + +// GetTumnbailPath returns the relative path to the thumbnail of a bookmark in the filesystem +func GetThumbnailPath(bookmark *BookmarkDTO) string { + return filepath.Join("thumb", strconv.Itoa(bookmark.ID)) +} + +// GetEbookPath returns the relative path to the ebook of a bookmark in the filesystem +func GetEbookPath(bookmark *BookmarkDTO) string { + return filepath.Join("ebook", strconv.Itoa(bookmark.ID)+".epub") +} + +// GetArchivePath returns the relative path to the archive of a bookmark in the filesystem +func GetArchivePath(bookmark *BookmarkDTO) string { + return filepath.Join("archive", strconv.Itoa(bookmark.ID)) } diff --git a/internal/model/domains.go b/internal/model/domains.go index 87a7d00aa..e2cb8e6d8 100644 --- a/internal/model/domains.go +++ b/internal/model/domains.go @@ -2,19 +2,18 @@ package model import ( "context" - "html/template" + "io/fs" "time" "github.com/go-shiori/warc" + "github.com/spf13/afero" ) type BookmarksDomain interface { HasEbook(b *BookmarkDTO) bool HasArchive(b *BookmarkDTO) bool - GetThumbnailPath(b *BookmarkDTO) string HasThumbnail(b *BookmarkDTO) bool GetBookmark(ctx context.Context, id DBID) (*BookmarkDTO, error) - GetBookmarkContentsFromArchive(bookmark *BookmarkDTO) (template.HTML, error) } type AccountsDomain interface { @@ -29,6 +28,8 @@ type ArchiverDomain interface { } type StorageDomain interface { + Stat(name string) (fs.FileInfo, error) + FS() afero.Fs FileExists(path string) bool DirExists(path string) bool } diff --git a/internal/testutil/shiori.go b/internal/testutil/shiori.go index 9fe96d386..86120658f 100644 --- a/internal/testutil/shiori.go +++ b/internal/testutil/shiori.go @@ -12,10 +12,13 @@ import ( "github.com/go-shiori/shiori/internal/model" "github.com/gofrs/uuid/v5" "github.com/sirupsen/logrus" + "github.com/spf13/afero" "github.com/stretchr/testify/require" ) func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logger *logrus.Logger) (*config.Config, *dependencies.Dependencies) { + t.Helper() + tmp, err := os.CreateTemp("", "") require.NoError(t, err) @@ -36,7 +39,7 @@ func GetTestConfigurationAndDependencies(t *testing.T, ctx context.Context, logg deps.Domains.Auth = domains.NewAccountsDomain(deps) deps.Domains.Archiver = domains.NewArchiverDomain(deps) deps.Domains.Bookmarks = domains.NewBookmarksDomain(deps) - deps.Domains.Storage = domains.NewStorageDomain(deps, os.DirFS(cfg.Storage.DataDir)) + deps.Domains.Storage = domains.NewStorageDomain(deps, afero.NewBasePathFs(afero.NewOsFs(), cfg.Storage.DataDir)) return cfg, deps } diff --git a/internal/webserver/handler-api-ext.go b/internal/webserver/handler-api-ext.go index 1a936b5d9..cf9270490 100644 --- a/internal/webserver/handler-api-ext.go +++ b/internal/webserver/handler-api-ext.go @@ -96,7 +96,7 @@ func (h *Handler) ApiInsertViaExtension(w http.ResponseWriter, r *http.Request, } var isFatalErr bool - book, isFatalErr, err = core.ProcessBookmark(request) + book, isFatalErr, err = core.ProcessBookmark(h.dependencies, request) if tmp, ok := contentBuffer.(io.ReadCloser); ok { tmp.Close() diff --git a/internal/webserver/handler-api.go b/internal/webserver/handler-api.go index 85a83e092..3772459bd 100644 --- a/internal/webserver/handler-api.go +++ b/internal/webserver/handler-api.go @@ -15,12 +15,13 @@ import ( "github.com/go-shiori/shiori/internal/core" "github.com/go-shiori/shiori/internal/database" + "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/model" "github.com/julienschmidt/httprouter" "golang.org/x/crypto/bcrypt" ) -func downloadBookmarkContent(book *model.BookmarkDTO, dataDir string, request *http.Request, keepTitle, keepExcerpt bool) (*model.BookmarkDTO, error) { +func downloadBookmarkContent(deps *dependencies.Dependencies, book *model.BookmarkDTO, dataDir string, request *http.Request, keepTitle, keepExcerpt bool) (*model.BookmarkDTO, error) { content, contentType, err := core.DownloadBookmark(book.URL) if err != nil { return nil, fmt.Errorf("error downloading url: %s", err) @@ -35,7 +36,7 @@ func downloadBookmarkContent(book *model.BookmarkDTO, dataDir string, request *h KeepExcerpt: keepExcerpt, } - result, isFatalErr, err := core.ProcessBookmark(processRequest) + result, isFatalErr, err := core.ProcessBookmark(deps, processRequest) content.Close() if err != nil && isFatalErr { @@ -239,7 +240,7 @@ func (h *Handler) ApiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h if payload.Async { go func() { - bookmark, err := downloadBookmarkContent(book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "") + bookmark, err := downloadBookmarkContent(h.dependencies, book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "") if err != nil { log.Printf("error downloading boorkmark: %s", err) return @@ -251,7 +252,7 @@ func (h *Handler) ApiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h } else { // Workaround. Download content after saving the bookmark so we have the proper database // id already set in the object regardless of the database engine. - book, err = downloadBookmarkContent(book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "") + book, err = downloadBookmarkContent(h.dependencies, book, h.DataDir, r, userHasDefinedTitle, book.Excerpt != "") if err != nil { log.Printf("error downloading boorkmark: %s", err) } else if _, err := h.DB.SaveBookmarks(ctx, false, *book); err != nil { From 692b1570ae004ffb82dafe6aa125a72d8792ea67 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Mon, 25 Dec 2023 12:21:19 +0100 Subject: [PATCH 17/20] removed mock to avoid increasing complexity --- internal/database/database.go | 1 - internal/domains/bookmarks_test.go | 19 +-- internal/mocks/database.go | 266 ----------------------------- 3 files changed, 8 insertions(+), 278 deletions(-) delete mode 100644 internal/mocks/database.go diff --git a/internal/database/database.go b/internal/database/database.go index 6bf210ac0..d79c56ca0 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,4 +1,3 @@ -//go:generate mockgen -destination=../mocks/database.go -package=mocks github.com/go-shiori/shiori/internal/database DB package database import ( diff --git a/internal/domains/bookmarks_test.go b/internal/domains/bookmarks_test.go index 5b3d724b0..950c7851a 100644 --- a/internal/domains/bookmarks_test.go +++ b/internal/domains/bookmarks_test.go @@ -5,24 +5,25 @@ import ( "testing" "github.com/go-shiori/shiori/internal/config" + "github.com/go-shiori/shiori/internal/database" "github.com/go-shiori/shiori/internal/dependencies" "github.com/go-shiori/shiori/internal/domains" - "github.com/go-shiori/shiori/internal/mocks" "github.com/go-shiori/shiori/internal/model" + "github.com/go-shiori/shiori/internal/testutil" "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" ) func TestBookmarkDomain(t *testing.T) { fs := afero.NewMemMapFs() - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() + db, err := database.OpenSQLiteDatabase(context.TODO(), ":memory:") + require.NoError(t, err) + require.NoError(t, db.Migrate()) deps := &dependencies.Dependencies{ - Database: mocks.NewMockDB(mockCtrl), + Database: db, Config: config.ParseServerConfiguration(context.TODO(), logrus.New()), Log: logrus.New(), Domains: &dependencies.Domains{}, @@ -67,12 +68,8 @@ func TestBookmarkDomain(t *testing.T) { t.Run("GetBookmark", func(t *testing.T) { t.Run("Success", func(t *testing.T) { - deps.Database.(*mocks.MockDB).EXPECT(). - GetBookmark(gomock.Any(), 1, ""). - Return(model.BookmarkDTO{ - ID: 1, - HTML: "

hello world

", - }, true, nil) + _, err := deps.Database.SaveBookmarks(context.TODO(), true, *testutil.GetValidBookmark()) + require.NoError(t, err) bookmark, err := domain.GetBookmark(context.Background(), 1) require.NoError(t, err) require.Equal(t, 1, bookmark.ID) diff --git a/internal/mocks/database.go b/internal/mocks/database.go deleted file mode 100644 index 6f0fc04fe..000000000 --- a/internal/mocks/database.go +++ /dev/null @@ -1,266 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/go-shiori/shiori/internal/database (interfaces: DB) -// -// Generated by this command: -// -// mockgen -destination=../mocks/database.go -package=mocks github.com/go-shiori/shiori/internal/database DB -// -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - database "github.com/go-shiori/shiori/internal/database" - model "github.com/go-shiori/shiori/internal/model" - gomock "go.uber.org/mock/gomock" -) - -// MockDB is a mock of DB interface. -type MockDB struct { - ctrl *gomock.Controller - recorder *MockDBMockRecorder -} - -// MockDBMockRecorder is the mock recorder for MockDB. -type MockDBMockRecorder struct { - mock *MockDB -} - -// NewMockDB creates a new mock instance. -func NewMockDB(ctrl *gomock.Controller) *MockDB { - mock := &MockDB{ctrl: ctrl} - mock.recorder = &MockDBMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockDB) EXPECT() *MockDBMockRecorder { - return m.recorder -} - -// CreateTags mocks base method. -func (m *MockDB) CreateTags(arg0 context.Context, arg1 ...model.Tag) error { - m.ctrl.T.Helper() - varargs := []any{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "CreateTags", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateTags indicates an expected call of CreateTags. -func (mr *MockDBMockRecorder) CreateTags(arg0 any, arg1 ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTags", reflect.TypeOf((*MockDB)(nil).CreateTags), varargs...) -} - -// DeleteAccounts mocks base method. -func (m *MockDB) DeleteAccounts(arg0 context.Context, arg1 ...string) error { - m.ctrl.T.Helper() - varargs := []any{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "DeleteAccounts", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAccounts indicates an expected call of DeleteAccounts. -func (mr *MockDBMockRecorder) DeleteAccounts(arg0 any, arg1 ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccounts", reflect.TypeOf((*MockDB)(nil).DeleteAccounts), varargs...) -} - -// DeleteBookmarks mocks base method. -func (m *MockDB) DeleteBookmarks(arg0 context.Context, arg1 ...int) error { - m.ctrl.T.Helper() - varargs := []any{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "DeleteBookmarks", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteBookmarks indicates an expected call of DeleteBookmarks. -func (mr *MockDBMockRecorder) DeleteBookmarks(arg0 any, arg1 ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBookmarks", reflect.TypeOf((*MockDB)(nil).DeleteBookmarks), varargs...) -} - -// GetAccount mocks base method. -func (m *MockDB) GetAccount(arg0 context.Context, arg1 string) (model.Account, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAccount", arg0, arg1) - ret0, _ := ret[0].(model.Account) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetAccount indicates an expected call of GetAccount. -func (mr *MockDBMockRecorder) GetAccount(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockDB)(nil).GetAccount), arg0, arg1) -} - -// GetAccounts mocks base method. -func (m *MockDB) GetAccounts(arg0 context.Context, arg1 database.GetAccountsOptions) ([]model.Account, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAccounts", arg0, arg1) - ret0, _ := ret[0].([]model.Account) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAccounts indicates an expected call of GetAccounts. -func (mr *MockDBMockRecorder) GetAccounts(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccounts", reflect.TypeOf((*MockDB)(nil).GetAccounts), arg0, arg1) -} - -// GetBookmark mocks base method. -func (m *MockDB) GetBookmark(arg0 context.Context, arg1 int, arg2 string) (model.BookmarkDTO, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBookmark", arg0, arg1, arg2) - ret0, _ := ret[0].(model.BookmarkDTO) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetBookmark indicates an expected call of GetBookmark. -func (mr *MockDBMockRecorder) GetBookmark(arg0, arg1, arg2 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBookmark", reflect.TypeOf((*MockDB)(nil).GetBookmark), arg0, arg1, arg2) -} - -// GetBookmarks mocks base method. -func (m *MockDB) GetBookmarks(arg0 context.Context, arg1 database.GetBookmarksOptions) ([]model.BookmarkDTO, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBookmarks", arg0, arg1) - ret0, _ := ret[0].([]model.BookmarkDTO) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBookmarks indicates an expected call of GetBookmarks. -func (mr *MockDBMockRecorder) GetBookmarks(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBookmarks", reflect.TypeOf((*MockDB)(nil).GetBookmarks), arg0, arg1) -} - -// GetBookmarksCount mocks base method. -func (m *MockDB) GetBookmarksCount(arg0 context.Context, arg1 database.GetBookmarksOptions) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBookmarksCount", arg0, arg1) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBookmarksCount indicates an expected call of GetBookmarksCount. -func (mr *MockDBMockRecorder) GetBookmarksCount(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBookmarksCount", reflect.TypeOf((*MockDB)(nil).GetBookmarksCount), arg0, arg1) -} - -// GetTags mocks base method. -func (m *MockDB) GetTags(arg0 context.Context) ([]model.Tag, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTags", arg0) - ret0, _ := ret[0].([]model.Tag) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTags indicates an expected call of GetTags. -func (mr *MockDBMockRecorder) GetTags(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTags", reflect.TypeOf((*MockDB)(nil).GetTags), arg0) -} - -// Migrate mocks base method. -func (m *MockDB) Migrate() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Migrate") - ret0, _ := ret[0].(error) - return ret0 -} - -// Migrate indicates an expected call of Migrate. -func (mr *MockDBMockRecorder) Migrate() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Migrate", reflect.TypeOf((*MockDB)(nil).Migrate)) -} - -// RenameTag mocks base method. -func (m *MockDB) RenameTag(arg0 context.Context, arg1 int, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RenameTag", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// RenameTag indicates an expected call of RenameTag. -func (mr *MockDBMockRecorder) RenameTag(arg0, arg1, arg2 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenameTag", reflect.TypeOf((*MockDB)(nil).RenameTag), arg0, arg1, arg2) -} - -// SaveAccount mocks base method. -func (m *MockDB) SaveAccount(arg0 context.Context, arg1 model.Account) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SaveAccount", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// SaveAccount indicates an expected call of SaveAccount. -func (mr *MockDBMockRecorder) SaveAccount(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveAccount", reflect.TypeOf((*MockDB)(nil).SaveAccount), arg0, arg1) -} - -// SaveAccountSettings mocks base method. -func (m *MockDB) SaveAccountSettings(arg0 context.Context, arg1 model.Account) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SaveAccountSettings", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// SaveAccountSettings indicates an expected call of SaveAccountSettings. -func (mr *MockDBMockRecorder) SaveAccountSettings(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveAccountSettings", reflect.TypeOf((*MockDB)(nil).SaveAccountSettings), arg0, arg1) -} - -// SaveBookmarks mocks base method. -func (m *MockDB) SaveBookmarks(arg0 context.Context, arg1 bool, arg2 ...model.BookmarkDTO) ([]model.BookmarkDTO, error) { - m.ctrl.T.Helper() - varargs := []any{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SaveBookmarks", varargs...) - ret0, _ := ret[0].([]model.BookmarkDTO) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SaveBookmarks indicates an expected call of SaveBookmarks. -func (mr *MockDBMockRecorder) SaveBookmarks(arg0, arg1 any, arg2 ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveBookmarks", reflect.TypeOf((*MockDB)(nil).SaveBookmarks), varargs...) -} From 467e2968e81b99603a1c82459cec043da4344082 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Mon, 25 Dec 2023 20:30:17 +0100 Subject: [PATCH 18/20] using deps to copy files around --- internal/core/ebook.go | 2 +- internal/core/processing.go | 37 +++----------------- internal/core/processing_test.go | 6 ++-- internal/domains/storage.go | 47 +++++++++++++++++++++----- internal/domains/storage_test.go | 58 ++++++++++++++++++++++++++++++++ internal/model/domains.go | 3 ++ 6 files changed, 108 insertions(+), 45 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 40cbef4aa..b03e3974e 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -76,7 +76,7 @@ func GenerateEbook(deps *dependencies.Dependencies, req ProcessRequest, dstPath defer tmpFile.Close() // If everything go well we move ebook to dstPath - err = MoveFileToDestination(deps.Domains.Storage.FS(), dstPath, tmpFile) + err = deps.Domains.Storage.WriteFile(dstPath, tmpFile) if err != nil { return book, errors.Wrap(err, "failed move ebook to destination") } diff --git a/internal/core/processing.go b/internal/core/processing.go index c97026c27..32a4837ae 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -22,7 +22,6 @@ import ( "github.com/go-shiori/shiori/internal/model" "github.com/go-shiori/warc" "github.com/pkg/errors" - "github.com/spf13/afero" _ "golang.org/x/image/webp" // Add support for png @@ -125,7 +124,7 @@ func ProcessBookmark(deps *dependencies.Dependencies, req ProcessRequest) (book // Save article image to local disk for i, imageURL := range imageURLs { - err = DownloadBookImage(deps.Domains.Storage.FS(), imageURL, imgPath) + err = DownloadBookImage(deps, imageURL, imgPath) if err != nil && errors.Is(err, ErrNoSupportedImageType) { log.Printf("%s: %s", err, imageURL) if i == len(imageURLs)-1 { @@ -181,7 +180,7 @@ func ProcessBookmark(deps *dependencies.Dependencies, req ProcessRequest) (book } dstPath := model.GetArchivePath(&book) - err = MoveFileToDestination(deps.Domains.Storage.FS(), dstPath, tmpFile) + err = deps.Domains.Storage.WriteFile(dstPath, tmpFile) if err != nil { return book, false, fmt.Errorf("failed move archive to destination `: %v", err) } @@ -192,7 +191,7 @@ func ProcessBookmark(deps *dependencies.Dependencies, req ProcessRequest) (book return book, false, nil } -func DownloadBookImage(fs afero.Fs, url, dstPath string) error { +func DownloadBookImage(deps *dependencies.Dependencies, url, dstPath string) error { // Fetch data from URL resp, err := httpClient.Get(url) if err != nil { @@ -264,38 +263,10 @@ func DownloadBookImage(fs afero.Fs, url, dstPath string) error { return fmt.Errorf("failed to save image %s: %v", url, err) } - err = MoveFileToDestination(fs, dstPath, tmpFile) + err = deps.Domains.Storage.WriteFile(dstPath, tmpFile) if err != nil { return err } return nil } - -// dstPath requires the filename -// TODO: move to storage domain -func MoveFileToDestination(fs afero.Fs, dstPath string, tmpFile *os.File) error { - // Prepare destination file. - err := fs.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) - if err != nil { - return fmt.Errorf("failed to create destination dir: %v", err) - } - - dstFile, err := fs.Create(dstPath) - if err != nil { - return fmt.Errorf("failed to create destination file: %v", err) - } - defer dstFile.Close() - // Copy temporary file to destination - _, err = tmpFile.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("failed to rewind temporary file: %v", err) - } - - _, err = io.Copy(dstFile, tmpFile) - if err != nil { - return fmt.Errorf("failed to copy file to the destination") - } - - return nil -} diff --git a/internal/core/processing_test.go b/internal/core/processing_test.go index 531eb1550..0e24f6007 100644 --- a/internal/core/processing_test.go +++ b/internal/core/processing_test.go @@ -29,7 +29,7 @@ func TestDownloadBookImage(t *testing.T) { defer os.Remove(dstPath) // Act - err := core.DownloadBookImage(deps.Domains.Storage.FS(), imageURL, dstPath) + err := core.DownloadBookImage(deps, imageURL, dstPath) // Assert assert.EqualError(t, err, "unsupported image type") @@ -43,7 +43,7 @@ func TestDownloadBookImage(t *testing.T) { defer os.Remove(dstPath) // Act - err := core.DownloadBookImage(deps.Domains.Storage.FS(), imageURL, dstPath) + err := core.DownloadBookImage(deps, imageURL, dstPath) // Assert assert.NoError(t, err) @@ -64,7 +64,7 @@ func TestDownloadBookImage(t *testing.T) { defer os.Remove(dstPath) // Act - err := core.DownloadBookImage(deps.Domains.Storage.FS(), imageURL, dstPath) + err := core.DownloadBookImage(deps, imageURL, dstPath) // Assert assert.NoError(t, err) diff --git a/internal/domains/storage.go b/internal/domains/storage.go index 186d031f3..9cb49dd40 100644 --- a/internal/domains/storage.go +++ b/internal/domains/storage.go @@ -1,11 +1,14 @@ package domains import ( + "fmt" + "io" "io/fs" "os" "path/filepath" "github.com/go-shiori/shiori/internal/dependencies" + "github.com/go-shiori/shiori/internal/model" "github.com/spf13/afero" ) @@ -14,6 +17,13 @@ type StorageDomain struct { fs afero.Fs } +func NewStorageDomain(deps *dependencies.Dependencies, fs afero.Fs) *StorageDomain { + return &StorageDomain{ + deps: deps, + fs: fs, + } +} + // Stat returns the FileInfo structure describing file. func (d *StorageDomain) Stat(name string) (fs.FileInfo, error) { return d.fs.Stat(name) @@ -36,11 +46,11 @@ func (d *StorageDomain) DirExists(name string) bool { return err == nil && info.IsDir() } -// Write writes data to a file in storage. +// WriteData writes bytes data to a file in storage. // CAUTION: This function will overwrite existing file. -func (d *StorageDomain) Save(name string, data []byte) error { +func (d *StorageDomain) WriteData(dst string, data []byte) error { // Create directory if not exist - dir := filepath.Dir(name) + dir := filepath.Dir(dst) if !d.DirExists(dir) { err := d.fs.MkdirAll(dir, os.ModePerm) if err != nil { @@ -49,7 +59,7 @@ func (d *StorageDomain) Save(name string, data []byte) error { } // Create file - file, err := d.fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + file, err := d.fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) if err != nil { return err } @@ -60,9 +70,30 @@ func (d *StorageDomain) Save(name string, data []byte) error { return err } -func NewStorageDomain(deps *dependencies.Dependencies, fs afero.Fs) *StorageDomain { - return &StorageDomain{ - deps: deps, - fs: fs, +// WriteFile writes a file to storage. +func (d *StorageDomain) WriteFile(dst string, tmpFile *os.File) error { + if dst != "" && !d.DirExists(dst) { + err := d.fs.MkdirAll(filepath.Dir(dst), model.DataDirPerm) + if err != nil { + return fmt.Errorf("failed to create destination dir: %v", err) + } + } + + dstFile, err := d.fs.Create(dst) + if err != nil { + return fmt.Errorf("failed to create destination file: %v", err) } + defer dstFile.Close() + + _, err = tmpFile.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("failed to rewind temporary file: %v", err) + } + + _, err = io.Copy(dstFile, tmpFile) + if err != nil { + return fmt.Errorf("failed to copy file to the destination") + } + + return nil } diff --git a/internal/domains/storage_test.go b/internal/domains/storage_test.go index 7e6a28038..969441ab1 100644 --- a/internal/domains/storage_test.go +++ b/internal/domains/storage_test.go @@ -2,6 +2,7 @@ package domains_test import ( "context" + "os" "testing" "github.com/go-shiori/shiori/internal/config" @@ -45,3 +46,60 @@ func TestFileExists(t *testing.T) { require.True(t, domain.FileExists("foo/file")) require.False(t, domain.FileExists("bar")) } + +func TestWriteFile(t *testing.T) { + fs := afero.NewMemMapFs() + + domain := domains.NewStorageDomain( + &dependencies.Dependencies{ + Config: config.ParseServerConfiguration(context.TODO(), logrus.New()), + Log: logrus.New(), + }, + fs, + ) + + err := domain.WriteData("foo/file.ext", []byte("foo")) + require.NoError(t, err) + require.True(t, domain.FileExists("foo/file.ext")) + require.True(t, domain.DirExists("foo")) + handler, err := domain.FS().Open("foo/file.ext") + require.NoError(t, err) + defer handler.Close() + + data, err := afero.ReadAll(handler) + require.NoError(t, err) + + require.Equal(t, "foo", string(data)) +} + +func TestSaveFile(t *testing.T) { + fs := afero.NewMemMapFs() + + domain := domains.NewStorageDomain( + &dependencies.Dependencies{ + Config: config.ParseServerConfiguration(context.TODO(), logrus.New()), + Log: logrus.New(), + }, + fs, + ) + + tempFile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tempFile.Name()) + + _, err = tempFile.WriteString("foo") + require.NoError(t, err) + + err = domain.WriteFile("foo/file.ext", tempFile) + require.NoError(t, err) + require.True(t, domain.FileExists("foo/file.ext")) + require.True(t, domain.DirExists("foo")) + handler, err := domain.FS().Open("foo/file.ext") + require.NoError(t, err) + defer handler.Close() + + data, err := afero.ReadAll(handler) + require.NoError(t, err) + + require.Equal(t, "foo", string(data)) +} diff --git a/internal/model/domains.go b/internal/model/domains.go index e2cb8e6d8..ce4e0532b 100644 --- a/internal/model/domains.go +++ b/internal/model/domains.go @@ -3,6 +3,7 @@ package model import ( "context" "io/fs" + "os" "time" "github.com/go-shiori/warc" @@ -32,4 +33,6 @@ type StorageDomain interface { FS() afero.Fs FileExists(path string) bool DirExists(path string) bool + WriteData(dst string, data []byte) error + WriteFile(dst string, src *os.File) error } From 230fab4ebc21a0309e50f3cd9874cf9bd237455f Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Mon, 25 Dec 2023 20:35:26 +0100 Subject: [PATCH 19/20] remove config usage (to deps) --- internal/http/routes/api/v1/bookmarks.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/http/routes/api/v1/bookmarks.go b/internal/http/routes/api/v1/bookmarks.go index 6d75c1eee..acf23521b 100644 --- a/internal/http/routes/api/v1/bookmarks.go +++ b/internal/http/routes/api/v1/bookmarks.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/gin-gonic/gin" - "github.com/go-shiori/shiori/internal/config" "github.com/go-shiori/shiori/internal/core" "github.com/go-shiori/shiori/internal/database" "github.com/go-shiori/shiori/internal/dependencies" @@ -213,10 +212,6 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) { return } - // Get server config - logger := logrus.New() - cfg := config.ParseServerConfiguration(ctx, logger) - var payload updateCachePayload if err := c.ShouldBindJSON(&payload); err != nil { response.SendInternalServerError(c) @@ -280,7 +275,7 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) { } request := core.ProcessRequest{ - DataDir: cfg.Storage.DataDir, + DataDir: r.deps.Config.Storage.DataDir, Bookmark: book, Content: content, ContentType: contentType, From dc1ee4821dd6bb9a5d2e42faa12804fd9147bcd2 Mon Sep 17 00:00:00 2001 From: "Felipe M." Date: Mon, 25 Dec 2023 22:03:09 +0100 Subject: [PATCH 20/20] remove handler-ui file --- internal/webserver/handler-ui.go | 112 ------------------------------- 1 file changed, 112 deletions(-) delete mode 100644 internal/webserver/handler-ui.go diff --git a/internal/webserver/handler-ui.go b/internal/webserver/handler-ui.go deleted file mode 100644 index 998a12476..000000000 --- a/internal/webserver/handler-ui.go +++ /dev/null @@ -1,112 +0,0 @@ -package webserver - -import ( - "bytes" - "compress/gzip" - "fmt" - "log" - "net/http" - "path" - fp "path/filepath" - "strconv" - "strings" - - "github.com/PuerkitoBio/goquery" - "github.com/go-shiori/warc" - "github.com/julienschmidt/httprouter" -) - -// ServeBookmarkArchive is handler for GET /bookmark/:id/archive/*filepath -func (h *Handler) ServeBookmarkArchive(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - ctx := r.Context() - - // Get parameter from URL - strID := ps.ByName("id") - resourcePath := ps.ByName("filepath") - resourcePath = strings.TrimPrefix(resourcePath, "/") - - // Get bookmark from database - id, err := strconv.Atoi(strID) - checkError(err) - - bookmark, exist, err := h.DB.GetBookmark(ctx, id, "") - checkError(err) - - if !exist { - panic(fmt.Errorf("bookmark not found")) - } - - // If it's not public, make sure session still valid - if bookmark.Public != 1 { - err = h.validateSession(r) - if err != nil { - newPath := path.Join(h.RootPath, "/login") - redirectURL := createRedirectURL(newPath, r.URL.String()) - redirectPage(w, r, redirectURL) - return - } - } - - // Open archive, look in cache first - var archive *warc.Archive - cacheData, found := h.ArchiveCache.Get(strID) - - if found { - archive = cacheData.(*warc.Archive) - } else { - archivePath := fp.Join(h.DataDir, "archive", strID) - archive, err = warc.Open(archivePath) - checkError(err) - - h.ArchiveCache.Set(strID, archive, 0) - } - - content, contentType, err := archive.Read(resourcePath) - checkError(err) - - // Set response header - w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Content-Type", contentType) - - // If this is HTML and root, inject shiori header - if strings.Contains(strings.ToLower(contentType), "text/html") && resourcePath == "" { - // Extract gzip - buffer := bytes.NewBuffer(content) - gzipReader, err := gzip.NewReader(buffer) - checkError(err) - - // Parse gzipped content - doc, err := goquery.NewDocumentFromReader(gzipReader) - checkError(err) - - // Add Shiori overlay - tplOutput := bytes.NewBuffer(nil) - err = h.templates["archive"].Execute(tplOutput, &bookmark) - checkError(err) - - archiveCSSPath := path.Join(h.RootPath, "/assets/css/archive.css") - - docHead := doc.Find("head") - docHead.PrependHtml(``) - docHead.AppendHtml(``) - doc.Find("body").PrependHtml(tplOutput.String()) - doc.Find("body").AddClass("shiori-archive-content") - - // Revert back to HTML - outerHTML, err := goquery.OuterHtml(doc.Selection) - checkError(err) - - // Gzip it again and send to response writer - gzipWriter := gzip.NewWriter(w) - if _, err := gzipWriter.Write([]byte(outerHTML)); err != nil { - log.Printf("error writing gzip file: %s", err) - } - gzipWriter.Flush() - return - } - - // Serve content - if _, err := w.Write(content); err != nil { - log.Printf("error writing response: %s", err) - } -}