From 10a6275e0ff6ad385c95fe615c279242352765b9 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:50:01 +0330 Subject: [PATCH 01/17] fix apple books cannot import generated book with shiori fix #638 --- internal/core/ebook.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 522094185..ea6b82817 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -102,8 +102,9 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) { } _, err = contentOpfWriter.Write([]byte(` - + ` + book.Title + ` + BookId From bb8c588d7e341d6c473255e22cc912ee96dde324 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:41:19 +0330 Subject: [PATCH 02/17] generate random content uuid for ebook fix #678 --- internal/core/ebook.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index ea6b82817..6ce9d275b 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -5,12 +5,14 @@ import ( "fmt" "io" "log" + "math/rand" "net/http" "os" fp "path/filepath" "regexp" "strconv" "strings" + "time" "github.com/go-shiori/shiori/internal/model" "github.com/pkg/errors" @@ -60,6 +62,8 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) { } } // create epub file + ebookIdentifier := GenerateEPUBIdentifier() + epubFile, err := os.Create(ebookfile) if err != nil { return book, errors.Wrap(err, "can't create ebook") @@ -101,10 +105,10 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) { return book, errors.Wrap(err, "can't create content.opf") } _, err = contentOpfWriter.Write([]byte(` - + ` + book.Title + ` - BookId + ` + ebookIdentifier + ` @@ -150,7 +154,7 @@ img { "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd"> - + @@ -253,3 +257,23 @@ func GetImages(html string) (map[string]string, error) { return images, nil } + +func GenerateEPUBIdentifier() string { + rand.Seed(time.Now().UnixNano()) + + // Define the valid characters for the identifier + validChars := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_:." + + // Generate a random identifier of length 24 + identifier := make([]byte, 24) + for i := range identifier { + identifier[i] = validChars[rand.Intn(len(validChars))] + } + + // Ensure the identifier starts with a letter + if identifier[0] >= '0' && identifier[0] <= '9' { + identifier[0] = validChars[rand.Intn(52)] // Generate a random letter + } + + return string(identifier) +} From b61266e15c26720a1dd90120ca58268de4540881 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Thu, 27 Jul 2023 15:35:18 +0330 Subject: [PATCH 03/17] replace rand.Seed() with new method for golang 1.20 --- internal/core/ebook.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 6ce9d275b..94a4b1cf8 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -259,7 +259,8 @@ func GetImages(html string) (map[string]string, error) { } func GenerateEPUBIdentifier() string { - rand.Seed(time.Now().UnixNano()) + src := rand.NewSource(time.Now().UnixNano()) + rnd := rand.New(src) // Define the valid characters for the identifier validChars := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_:." @@ -267,12 +268,12 @@ func GenerateEPUBIdentifier() string { // Generate a random identifier of length 24 identifier := make([]byte, 24) for i := range identifier { - identifier[i] = validChars[rand.Intn(len(validChars))] + identifier[i] = validChars[rnd.Intn(len(validChars))] } // Ensure the identifier starts with a letter if identifier[0] >= '0' && identifier[0] <= '9' { - identifier[0] = validChars[rand.Intn(52)] // Generate a random letter + identifier[0] = validChars[rnd.Intn(52)] // Generate a random letter } return string(identifier) From 49e8ba8f28f18d2921814edbaa54e9cefd965035 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Mon, 21 Aug 2023 00:54:14 +0330 Subject: [PATCH 04/17] use go-epub to generate ebook --- go.mod | 4 +- go.sum | 8 +- internal/core/ebook.go | 181 ++--------------------------------------- 3 files changed, 17 insertions(+), 176 deletions(-) diff --git a/go.mod b/go.mod index b3757992a..ce84a369a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/PuerkitoBio/goquery v1.8.1 + github.com/bmaupin/go-epub v1.1.1-0.20230816135150-45f45a4d57ba github.com/disintegration/imaging v1.6.2 github.com/fatih/color v1.15.0 github.com/gin-contrib/gzip v0.0.6 @@ -33,7 +34,7 @@ require ( github.com/swaggo/swag v1.16.1 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f golang.org/x/crypto v0.11.0 - golang.org/x/net v0.12.0 + golang.org/x/net v0.13.0 golang.org/x/term v0.10.0 modernc.org/sqlite v1.24.0 ) @@ -82,6 +83,7 @@ require ( github.com/tdewolff/parse v2.3.4+incompatible // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect diff --git a/go.sum b/go.sum index b6d074809..d61c0e139 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/bmaupin/go-epub v1.1.1-0.20230816135150-45f45a4d57ba h1:KsxwiuDsAixHe+PH0uVbRGs2EVVBxyfKF/Io67gV27A= +github.com/bmaupin/go-epub v1.1.1-0.20230816135150-45f45a4d57ba/go.mod h1:Wzh2rgIAWokCBSSF4C5/Z1LuufAGu/cETNd6O+2856E= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.2 h1:GDaNjuWSGu09guE9Oql0MSTNhNCLlWwO8y/xM5BzcbM= github.com/bytedance/sonic v1.9.2/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -233,6 +235,8 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= @@ -263,8 +267,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 6fd45464d..89ac206dc 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -1,17 +1,14 @@ package core import ( - "archive/zip" "fmt" - "io" - "log" - "net/http" "os" fp "path/filepath" "regexp" "strconv" "strings" + epub "github.com/bmaupin/go-epub" "github.com/go-shiori/shiori/internal/model" "github.com/pkg/errors" ) @@ -21,7 +18,7 @@ import ( // The bookmark model will be used to update the UI based on whether this function is successful or not. func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) { // variable for store generated html code - var html string + //var html string book = req.Bookmark @@ -60,175 +57,13 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err defer os.Remove(tmpFile.Name()) // Create zip archive - epubWriter := zip.NewWriter(tmpFile) + //epubWriter := zip.NewWriter(tmpFile) + ebook := epub.NewEpub(book.Title) + ebook.SetTitle(book.Title) + ebook.AddSection(string(book.HTML), book.Title, "", "") + ebook.EmbedImages() + ebook.Write(tmpFile.Name()) - // Create the mimetype file - mimetypeWriter, err := epubWriter.Create("mimetype") - if err != nil { - return book, errors.Wrap(err, "can't create mimetype") - } - _, err = mimetypeWriter.Write([]byte("application/epub+zip")) - if err != nil { - return book, errors.Wrap(err, "can't write into mimetype file") - } - - // Create the container.xml file - containerWriter, err := epubWriter.Create("META-INF/container.xml") - if err != nil { - return book, errors.Wrap(err, "can't create container.xml") - } - - _, err = containerWriter.Write([]byte(` - - - - -`)) - if err != nil { - return book, errors.Wrap(err, "can't write into container.xml file") - } - - contentOpfWriter, err := epubWriter.Create("OEBPS/content.opf") - if err != nil { - return book, errors.Wrap(err, "can't create content.opf") - } - _, err = contentOpfWriter.Write([]byte(` - - - ` + book.Title + ` - - - - - - - - - -`)) - if err != nil { - return book, errors.Wrap(err, "can't write into container.opf file") - } - - // Create the style.css file - styleWriter, err := epubWriter.Create("style.css") - if err != nil { - return book, errors.Wrap(err, "can't create content.xml") - } - _, err = styleWriter.Write([]byte(`content { - display: block; - font-size: 1em; - line-height: 1.2; - padding-left: 0; - padding-right: 0; - text-align: justify; - margin: 0 5pt -} -img { - margin: auto; - display: block; -}`)) - if err != nil { - return book, errors.Wrap(err, "can't write into style.css file") - } - // Create the toc.ncx file - tocNcxWriter, err := epubWriter.Create("OEBPS/toc.ncx") - if err != nil { - return book, errors.Wrap(err, "can't create toc.ncx") - } - _, err = tocNcxWriter.Write([]byte(` - - - - - - - - - - ` + book.Title + ` - - - - - ` + book.Title + ` - - - - -`)) - if err != nil { - return book, errors.Wrap(err, "can't write into toc.ncx file") - } - - // get list of images tag in html - imageList, _ := GetImages(book.HTML) - imgRegex := regexp.MustCompile(``) - - // Create a set to store unique image URLs - imageSet := make(map[string]bool) - - // Download image in html file and generate new html - html = book.HTML - for _, match := range imgRegex.FindAllStringSubmatch(book.HTML, -1) { - imageURL := match[1] - if _, ok := imageList[imageURL]; ok && !imageSet[imageURL] { - // Add the image URL to the set - imageSet[imageURL] = true - - // Download the image - resp, err := http.Get(imageURL) - if err != nil { - log.Fatal(err) - } - defer resp.Body.Close() - - // Get the image data - imageData, err := io.ReadAll(resp.Body) - if err != nil { - return book, errors.Wrap(err, "can't get image from the internet") - } - - fileName := fp.Base(imageURL) - filePath := "images/" + fileName - imageWriter, err := epubWriter.Create(filePath) - if err != nil { - log.Fatal(err) - } - - // Write the image to the file - _, err = imageWriter.Write(imageData) - if err != nil { - return book, errors.Wrap(err, "can't create image file") - } - // Replace the image tag with the new downloaded image - html = strings.ReplaceAll(html, match[0], fmt.Sprintf(``, filePath)) - } - } - // Create the content.html file - contentHtmlWriter, err := epubWriter.Create("OEBPS/content.html") - if err != nil { - return book, errors.Wrap(err, "can't create content.xml") - } - _, err = contentHtmlWriter.Write([]byte("\n\n\n\t" + book.Title + "\n\t\n\n\n\t

" + book.Title + "

" + "\n\n" + html + "\n" + "\n")) - if err != nil { - return book, errors.Wrap(err, "can't write into content.html") - } - // close epub and tmpFile - err = epubWriter.Close() - if err != nil { - return book, errors.Wrap(err, "failed to close EPUB writer") - } - err = tmpFile.Close() - if err != nil { - return book, errors.Wrap(err, "failed to close temporary EPUB file") - } - // open temporary file again - tmpFile, err = os.Open(tmpFile.Name()) - if err != nil { - return book, errors.Wrap(err, "can't open temporary EPUB file") - } defer tmpFile.Close() // if everitings go well we start move ebook to dstPath err = MoveFileToDestination(dstPath, tmpFile) From b6f4e16f83865e213d3c3a4aa09a67c301c41c4a Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 26 Aug 2023 14:19:38 +0330 Subject: [PATCH 05/17] add author and Description metadata --- internal/core/ebook.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 89ac206dc..b47d8e075 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -60,6 +60,8 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err //epubWriter := zip.NewWriter(tmpFile) ebook := epub.NewEpub(book.Title) ebook.SetTitle(book.Title) + ebook.SetAuthor(book.Author) + ebook.SetDescription(book.Excerpt) ebook.AddSection(string(book.HTML), book.Title, "", "") ebook.EmbedImages() ebook.Write(tmpFile.Name()) From d7bab48970852eb74cd45580a8624735101f1403 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 26 Aug 2023 16:31:35 +0330 Subject: [PATCH 06/17] cleanup old code --- internal/core/ebook.go | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index b47d8e075..422c521e4 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -4,7 +4,6 @@ import ( "fmt" "os" fp "path/filepath" - "regexp" "strconv" "strings" @@ -17,8 +16,6 @@ import ( // The destination path `dstPath` should include file name with ".epub" extension // The bookmark model will be used to update the UI based on whether this function is successful or not. func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) { - // variable for store generated html code - //var html string book = req.Bookmark @@ -27,8 +24,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err return book, errors.New("bookmark ID is not valid") } - // get current state of bookmark - // cheak archive and thumb + // get current state of bookmark cheak archive and thumb strID := strconv.Itoa(book.ID) imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID)) @@ -56,8 +52,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err } defer os.Remove(tmpFile.Name()) - // Create zip archive - //epubWriter := zip.NewWriter(tmpFile) + // Create ebook ebook := epub.NewEpub(book.Title) ebook.SetTitle(book.Title) ebook.SetAuthor(book.Author) @@ -67,6 +62,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err ebook.Write(tmpFile.Name()) defer tmpFile.Close() + // if everitings go well we start move ebook to dstPath err = MoveFileToDestination(dstPath, tmpFile) if err != nil { @@ -76,29 +72,3 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err book.HasEbook = true return book, nil } - -// function get html and return list of image url inside html file -func GetImages(html string) (map[string]string, error) { - // Regular expression to match image tags and their URLs - imageTagRegex := regexp.MustCompile(``) - - // Find all matches in the HTML string - imageTagMatches := imageTagRegex.FindAllStringSubmatch(html, -1) - // Create a dictionary to store the image URLs - images := make(map[string]string) - - // Check if there are any matches - if len(imageTagMatches) == 0 { - return nil, nil - } - - // Loop through all the matches and add them to the dictionary - for _, match := range imageTagMatches { - imageURL := match[1] - if !strings.HasPrefix(imageURL, "data:image/") { - images[imageURL] = match[0] - } - } - - return images, nil -} From cc4fbef5b78ac82f2ed7c779401741c32459a818 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 26 Aug 2023 19:38:00 +0330 Subject: [PATCH 07/17] remove unneeded unittests --- internal/core/ebook_test.go | 71 ------------------------------------- 1 file changed, 71 deletions(-) diff --git a/internal/core/ebook_test.go b/internal/core/ebook_test.go index 25b0eca0d..da7072021 100644 --- a/internal/core/ebook_test.go +++ b/internal/core/ebook_test.go @@ -171,74 +171,3 @@ func TestGenerateEbook(t *testing.T) { }) }) } - -// Add more unit tests for other scenarios that missing specialy -// can't create ebook directory and can't write situatuin -// writing inside zip file -// html variable that not export and image download loop - -func TestGetImages(t *testing.T) { - // Test case 1: HTML with no image tags - html1 := `

Hello, World!

` - expected1 := make(map[string]string) - result1, err1 := core.GetImages(html1) - if err1 != nil { - t.Errorf("Unexpected error: %v", err1) - } - if len(result1) != len(expected1) { - t.Errorf("Expected %d images, but got %d", len(expected1), len(result1)) - } - - // Test case 2: HTML with one image tag - html2 := `` - expected2 := map[string]string{"image1.jpg": ""} - result2, err2 := core.GetImages(html2) - if err2 != nil { - t.Errorf("Unexpected error: %v", err2) - } - if len(result2) != len(expected2) { - t.Errorf("Expected %d images, but got %d", len(expected2), len(result2)) - } - for key, value := range expected2 { - if result2[key] != value { - t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result2[key]) - } - } - - // Test case 3: HTML with multiple image tags - html3 := `` - expected3 := map[string]string{ - "image1.jpg": "", - "image2.jpg": "", - } - result3, err3 := core.GetImages(html3) - if err3 != nil { - t.Errorf("Unexpected error: %v", err3) - } - if len(result3) != len(expected3) { - t.Errorf("Expected %d images, but got %d", len(expected3), len(result3)) - } - for key, value := range expected3 { - if result3[key] != value { - t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result3[key]) - } - } - // Test case 4: HTML with multiple image tags with duplicayr - html4 := `` - expected4 := map[string]string{ - "image1.jpg": "", - "image2.jpg": "", - } - result4, err4 := core.GetImages(html4) - if err4 != nil { - t.Errorf("Unexpected error: %v", err4) - } - if len(result4) != len(expected4) { - t.Errorf("Expected %d images, but got %d", len(expected4), len(result4)) - } - for key, value := range expected4 { - if result4[key] != value { - t.Errorf("Expected image URL %s with tag %s, but got %s", key, value, result4[key]) - } - } -} From 17f00300c080dc6f8baea85773c4eae4ae8d677f Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:28:46 +0330 Subject: [PATCH 08/17] add url of bookmark to the end of ebook --- internal/core/ebook.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 422c521e4..d3dc37b50 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -24,7 +24,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err return book, errors.New("bookmark ID is not valid") } - // get current state of bookmark cheak archive and thumb + // Get current state of bookmark cheak archive and thumb strID := strconv.Itoa(book.ID) imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID)) @@ -38,32 +38,35 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err book.HasArchive = true } - // this function create ebook from reader mode of bookmark so + // This function create ebook from reader mode of bookmark so // we can't create ebook from PDF so we return error here if bookmark is a pdf contentType := req.ContentType if strings.Contains(contentType, "application/pdf") { return book, errors.New("can't create ebook for pdf") } - // create temporary epub file + // Create temporary epub file tmpFile, err := os.CreateTemp("", "ebook") if err != nil { return book, errors.Wrap(err, "can't create temporary EPUB file") } defer os.Remove(tmpFile.Name()) + // Create last line of ebook + lastline := `

Generated by Shiori From This Page

` + // Create ebook ebook := epub.NewEpub(book.Title) ebook.SetTitle(book.Title) ebook.SetAuthor(book.Author) ebook.SetDescription(book.Excerpt) - ebook.AddSection(string(book.HTML), book.Title, "", "") + ebook.AddSection(string(book.HTML+lastline), book.Title, "", "") ebook.EmbedImages() ebook.Write(tmpFile.Name()) defer tmpFile.Close() - // if everitings go well we start move ebook to dstPath + // If everitings go well we start move ebook to dstPath err = MoveFileToDestination(dstPath, tmpFile) if err != nil { return book, errors.Wrap(err, "failed move ebook to destination") From 73d6b728ea79f2271dc1f69021a5c2550f76d76b Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 27 Aug 2023 15:59:15 +0330 Subject: [PATCH 09/17] add title to ebook --- internal/core/ebook.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index d3dc37b50..eccef4bc6 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -53,14 +53,14 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err defer os.Remove(tmpFile.Name()) // Create last line of ebook - lastline := `

Generated by Shiori From This Page

` + lastline := `

Generated By Shiori From This Page

` // Create ebook ebook := epub.NewEpub(book.Title) ebook.SetTitle(book.Title) ebook.SetAuthor(book.Author) ebook.SetDescription(book.Excerpt) - ebook.AddSection(string(book.HTML+lastline), book.Title, "", "") + ebook.AddSection(string(`

`+book.Title+`

`+book.HTML+lastline), book.Title, "", "") ebook.EmbedImages() ebook.Write(tmpFile.Name()) From cdfbd500992c6279def0ae7bb5a8363210a39f49 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 27 Aug 2023 17:32:54 +0330 Subject: [PATCH 10/17] add shiori repository url and add hr tag before last line --- internal/core/ebook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index eccef4bc6..ac5502fe3 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -53,7 +53,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err defer os.Remove(tmpFile.Name()) // Create last line of ebook - lastline := `

Generated By Shiori From This Page

` + lastline := `

Generated By Shiori From This Page

` // Create ebook ebook := epub.NewEpub(book.Title) From 47ea6c120015b66de4af890e6fbb08f13962c277 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 27 Aug 2023 17:47:47 +0330 Subject: [PATCH 11/17] fix unneeded convert --- internal/core/ebook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index ac5502fe3..85ea9f755 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -60,7 +60,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err ebook.SetTitle(book.Title) ebook.SetAuthor(book.Author) ebook.SetDescription(book.Excerpt) - ebook.AddSection(string(`

`+book.Title+`

`+book.HTML+lastline), book.Title, "", "") + ebook.AddSection(`

`+book.Title+`

`+book.HTML+lastline, book.Title, "", "") ebook.EmbedImages() ebook.Write(tmpFile.Name()) From 15120206ae9c6aaf0d7362fac1fd6eff72c641c7 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Wed, 30 Aug 2023 19:02:28 +0330 Subject: [PATCH 12/17] use go-shiori/go-epub --- go.mod | 2 +- go.sum | 4 ++-- internal/core/ebook.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ce84a369a..a0d3eb0d2 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.20 require ( github.com/PuerkitoBio/goquery v1.8.1 - github.com/bmaupin/go-epub v1.1.1-0.20230816135150-45f45a4d57ba github.com/disintegration/imaging v1.6.2 github.com/fatih/color v1.15.0 github.com/gin-contrib/gzip v0.0.6 github.com/gin-contrib/requestid v0.0.6 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.9.1 + github.com/go-shiori/go-epub v1.1.1-0.20230829131709-922954663716 github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad github.com/go-shiori/warc v0.0.0-20200621032813-359908319d1d github.com/go-sql-driver/mysql v1.7.1 diff --git a/go.sum b/go.sum index d61c0e139..afed897c4 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= -github.com/bmaupin/go-epub v1.1.1-0.20230816135150-45f45a4d57ba h1:KsxwiuDsAixHe+PH0uVbRGs2EVVBxyfKF/Io67gV27A= -github.com/bmaupin/go-epub v1.1.1-0.20230816135150-45f45a4d57ba/go.mod h1:Wzh2rgIAWokCBSSF4C5/Z1LuufAGu/cETNd6O+2856E= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.2 h1:GDaNjuWSGu09guE9Oql0MSTNhNCLlWwO8y/xM5BzcbM= github.com/bytedance/sonic v1.9.2/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -76,6 +74,8 @@ github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QX github.com/go-shiori/dom v0.0.0-20190930082056-9d974a4f8b25/go.mod h1:360KoNl36ftFYhjLHuEty78kWUGw8i1opEicvIDLfRk= github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-epub v1.1.1-0.20230829131709-922954663716 h1:itLthh959oyRxPoWINQU/JL5FuQPqnCPBROgimyqs+8= +github.com/go-shiori/go-epub v1.1.1-0.20230829131709-922954663716/go.mod h1:whnnn+cj52BOHlqcWWYfKelM4xO31KfqxQQ1oo6h8Xg= github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad h1:3VP5Q8Mh165h2DHmXWFT4LJlwwvgTRlEuoe2vnsVnJ4= github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI= github.com/go-shiori/warc v0.0.0-20200621032813-359908319d1d h1:+SEf4hYDaAt2eyq8Xu3YyWCpnMsK8sZfbYsDRFCUgBM= diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 85ea9f755..29dc332fb 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - epub "github.com/bmaupin/go-epub" + epub "github.com/go-shiori/go-epub" "github.com/go-shiori/shiori/internal/model" "github.com/pkg/errors" ) From 09683d73fb901f3d0cc9ce1d08b302a8e9ef877f Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 16 Sep 2023 15:13:37 +0330 Subject: [PATCH 13/17] update go-epub to the last version --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a0d3eb0d2..4ad816efc 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gin-contrib/requestid v0.0.6 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.9.1 - github.com/go-shiori/go-epub v1.1.1-0.20230829131709-922954663716 + github.com/go-shiori/go-epub v1.1.1-0.20230916103239-8f86fbaf78c0 github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad github.com/go-shiori/warc v0.0.0-20200621032813-359908319d1d github.com/go-sql-driver/mysql v1.7.1 @@ -59,6 +59,7 @@ require ( github.com/go-playground/validator/v10 v10.14.1 // indirect github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/gofrs/uuid/v5 v5.0.0 // indirect github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index afed897c4..3bee35937 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QX github.com/go-shiori/dom v0.0.0-20190930082056-9d974a4f8b25/go.mod h1:360KoNl36ftFYhjLHuEty78kWUGw8i1opEicvIDLfRk= github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= -github.com/go-shiori/go-epub v1.1.1-0.20230829131709-922954663716 h1:itLthh959oyRxPoWINQU/JL5FuQPqnCPBROgimyqs+8= -github.com/go-shiori/go-epub v1.1.1-0.20230829131709-922954663716/go.mod h1:whnnn+cj52BOHlqcWWYfKelM4xO31KfqxQQ1oo6h8Xg= +github.com/go-shiori/go-epub v1.1.1-0.20230916103239-8f86fbaf78c0 h1:aZgu8Q4dbVG5QiGb2TsCERR2zC261+5gGqO59V/p/wk= +github.com/go-shiori/go-epub v1.1.1-0.20230916103239-8f86fbaf78c0/go.mod h1:gQCqrK+dIMLA7JMd8GxdBvhn811wb7XCa733RxWfPYw= github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad h1:3VP5Q8Mh165h2DHmXWFT4LJlwwvgTRlEuoe2vnsVnJ4= github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI= github.com/go-shiori/warc v0.0.0-20200621032813-359908319d1d h1:+SEf4hYDaAt2eyq8Xu3YyWCpnMsK8sZfbYsDRFCUgBM= @@ -88,6 +88,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid/v5 v5.0.0 h1:p544++a97kEL+svbcFbCQVM9KFu0Yo25UoISXGNNH9M= +github.com/gofrs/uuid/v5 v5.0.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= From 70d5518931e7e5c945b76123733a65be34e02bbf Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 16 Sep 2023 15:22:47 +0330 Subject: [PATCH 14/17] fix error handling --- internal/core/ebook.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 29dc332fb..9fd955b86 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -56,7 +56,11 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err lastline := `

Generated By Shiori From This Page

` // Create ebook - ebook := epub.NewEpub(book.Title) + ebook, err := epub.NewEpub(book.Title) + if err != nil { + return book, errors.Wrap(err, "can't create EPUB") + } + ebook.SetTitle(book.Title) ebook.SetAuthor(book.Author) ebook.SetDescription(book.Excerpt) From daf8aa240cc9d3d26ca6afbdb622beef1cef11b6 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 16 Sep 2023 16:28:17 +0330 Subject: [PATCH 15/17] add more error handling --- internal/core/ebook.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 9fd955b86..9fd224295 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -64,9 +64,15 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err ebook.SetTitle(book.Title) ebook.SetAuthor(book.Author) ebook.SetDescription(book.Excerpt) - ebook.AddSection(`

`+book.Title+`

`+book.HTML+lastline, book.Title, "", "") + _, err = ebook.AddSection(`

`+book.Title+`

`+book.HTML+lastline, book.Title, "", "") + if err != nil { + return book, errors.Wrap(err, "can't add ebook Section") + } ebook.EmbedImages() - ebook.Write(tmpFile.Name()) + err = ebook.Write(tmpFile.Name()) + if err != nil { + return book, errors.Wrap(err, "can't create ebook file") + } defer tmpFile.Close() From bad4a1d1c026d989749f19817e28b5d743556ebb Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 16 Sep 2023 21:01:45 +0330 Subject: [PATCH 16/17] go-epub v1.2.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 833a2b011..ae99f7795 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gin-contrib/requestid v0.0.6 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.9.1 - github.com/go-shiori/go-epub v1.1.1-0.20230916103239-8f86fbaf78c0 + github.com/go-shiori/go-epub v1.2.0 github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad github.com/go-shiori/warc v0.0.0-20200621032813-359908319d1d github.com/go-sql-driver/mysql v1.7.1 diff --git a/go.sum b/go.sum index 5116539f5..33bb5318e 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QX github.com/go-shiori/dom v0.0.0-20190930082056-9d974a4f8b25/go.mod h1:360KoNl36ftFYhjLHuEty78kWUGw8i1opEicvIDLfRk= github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= -github.com/go-shiori/go-epub v1.1.1-0.20230916103239-8f86fbaf78c0 h1:aZgu8Q4dbVG5QiGb2TsCERR2zC261+5gGqO59V/p/wk= -github.com/go-shiori/go-epub v1.1.1-0.20230916103239-8f86fbaf78c0/go.mod h1:gQCqrK+dIMLA7JMd8GxdBvhn811wb7XCa733RxWfPYw= +github.com/go-shiori/go-epub v1.2.0 h1:c2b3DblHpNIiD8ISlQ+0Mc/tsRmn1mX1l6Q/0LzavN4= +github.com/go-shiori/go-epub v1.2.0/go.mod h1:gQCqrK+dIMLA7JMd8GxdBvhn811wb7XCa733RxWfPYw= github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad h1:3VP5Q8Mh165h2DHmXWFT4LJlwwvgTRlEuoe2vnsVnJ4= github.com/go-shiori/go-readability v0.0.0-20230421032831-c66949dfc0ad/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI= github.com/go-shiori/warc v0.0.0-20200621032813-359908319d1d h1:+SEf4hYDaAt2eyq8Xu3YyWCpnMsK8sZfbYsDRFCUgBM= From 5a9e11db9038b55a5ba758901320c6e50aeab310 Mon Sep 17 00:00:00 2001 From: Monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 16 Sep 2023 21:49:15 +0330 Subject: [PATCH 17/17] better comment Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com> --- internal/core/ebook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 9fd224295..d3720cce1 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -76,7 +76,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err defer tmpFile.Close() - // If everitings go well we start move ebook to dstPath + // If everything go well we move ebook to dstPath err = MoveFileToDestination(dstPath, tmpFile) if err != nil { return book, errors.Wrap(err, "failed move ebook to destination")