Run legacy API and new API at the same time. (#648)

* config: backwards comptabile dir

* remove duplicated frontend

* frontend: move assets to assets folder

* legacy routes handler in gin

* templates and asset in different embed

* new routes

* frontend routes serve old views

* added DTO for account object

* api auth calls legacy handler

* frontend: handle new error messages

* frontend: update urls

* frontend: login using new api

* updated frontend tests

* chore: remove debug route

* create shiori/gopher user if no owner is present

* server as default command

* serve -> server

* refactored database logic, allow database url

* removed unused configuration

* storage docs

* refactor cli to use cfg and deps

* check errors only in server

* log fatal instead of os exit

* dont default data directory to current dir

* fixed sqlite path

* trigger build on prs

* avoid releasing if lint/test fails

* pull request condition

* event -> event_name

* Get correct pull request number

* added workflow to delete dangling tags

* fix: nil error checking

* set gin mode first

* set gin mode before initialization

* fix logger

* allow version bump from custom ref

* Updated matrix link to workspace

.github/workflows/_buildx.yml

@@ -23,10 +23,10 @@ jobs:
           mv binaries/shiori_linux_amd64_v1 binaries/shiori_linux_amd64
           gzip -d -S binaries/.gz__  -r .
           chmod 755 binaries/shiori_linux_*/shiori
-      - name: Buildx
-        working-directory: .github/workflows/docker
+      - name: Prepare master push tags
+        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
         run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login -u "${{ github.repository_owner }}" --password-stdin ghcr.io
+          echo "tag_flags=--tag ${{ github.ref }}" >> $GITHUB_ENV
           REPO=ghcr.io/${{ github.repository }}
           TAG=$(git describe --tags)
           if [ -z "$(git tag --points-at HEAD)" ]
@@ -35,5 +35,19 @@ jobs:
           else
             TAG2="latest"
           fi
+          echo "tag_flags=--tag $REPO:$TAG --tag $REPO:$TAG2" >> $GITHUB_ENV
+
+      - name: Prepare pull request tags
+        if: github.event_name == 'pull_request'
+        run: |
+          echo "tag_flags=--tag ${{ github.ref }}" >> $GITHUB_ENV
+          REPO=ghcr.io/${{ github.repository }}
+          echo "tag_flags=--tag $REPO:pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+
+      - name: Buildx
+        working-directory: .github/workflows/docker
+        run: |
+          set -x
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login -u "${{ github.repository_owner }}" --password-stdin ghcr.io
           docker buildx create --use --name builder
-          docker buildx build -f Dockerfile.ci --platform=linux/amd64,arm64,linux/arm/v7 --push --output=type=registry --tag $REPO:$TAG --tag $REPO:$TAG2 .
+          docker buildx build -f Dockerfile.ci --platform=linux/amd64,arm64,linux/arm/v7 --push ${{ env.tag_flags }} .

.github/workflows/_delete-registry-tag.yml

@@ -0,0 +1,14 @@
+name: Delete registry tag
+
+on: workflow_call
+
+jobs:
+  purge-image:
+    name: Delete tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: chipkent/action-cleanup-package@1316a66015b82d745b57acbb6c570f2bb1d108f9 # v1.0.3
+        with:
+          package-name: ${{ github.event.repository.name }}
+          tag: ${{ env.TAG_NAME }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docker/Dockerfile.ci

@@ -8,4 +8,4 @@ WORKDIR /shiori
 EXPOSE 8080
 ENV SHIORI_DIR=/shiori
 ENTRYPOINT ["/usr/bin/shiori"]
-CMD ["serve"]
+CMD ["server"]

.github/workflows/pull_request.yml

@@ -2,7 +2,8 @@ name: 'Pull Request'
 
 on:
   pull_request:
-    branches: master
+    branches:
+    - master
 
 concurrency:
   group: ci-tests-${{ github.ref }}-1
@@ -13,3 +14,9 @@ jobs:
     uses: ./.github/workflows/_golangci-lint.yml
   call-test:
     uses: ./.github/workflows/_test.yml
+  call-gorelease:
+    needs: [call-lint, call-test]
+    uses: ./.github/workflows/_gorelease.yml
+  call-buildx:
+    needs: call-gorelease
+    uses: ./.github/workflows/_buildx.yml

.github/workflows/pull_request_closed.yml

@@ -0,0 +1,12 @@
+name: 'Clean up Docker images from PR'
+
+on:
+  pull_request:
+    types:
+    - closed
+
+jobs:
+  delete-tag:
+    uses: ./.github/workflows/_delete-registry-tag.yml
+    with:
+      TAG_NAME: pr-${{ github.event.pull_request.number }}

.github/workflows/push.yml

@@ -15,6 +15,7 @@ jobs:
   call-test:
     uses: ./.github/workflows/_test.yml
   call-gorelease:
+    needs: [call-lint, call-test]
     uses: ./.github/workflows/_gorelease.yml
   call-buildx:
     needs: call-gorelease

.github/workflows/version_bump.yml

@@ -6,6 +6,11 @@ on:
       version:
         description: "Version to bump to, example: v1.5.2"
         required: true
+      ref:
+        description: "Ref to release from"
+        required: true
+        type: string
+        default: master
 
 jobs:
   tag-release:
@@ -18,7 +23,7 @@ jobs:
         uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
         with:
           fetch-depth: 0
-          ref: master
+          ref: ${{ inputs.ref }}
       - name: Tag release
         run: |
           git config user.email "${{github.repository_owner}}@users.noreply.github.com"

Dockerfile

@@ -16,4 +16,4 @@ WORKDIR /shiori
 EXPOSE 8080
 ENV SHIORI_DIR /shiori/
 ENTRYPOINT ["/usr/bin/shiori"]
-CMD ["serve"]
+CMD ["server"]

Dockerfile.compose

@@ -3,4 +3,4 @@ FROM docker.io/golang:1.19-alpine3.16
 WORKDIR /src/shiori
 
 ENTRYPOINT ["go", "run", "main.go"]
-CMD ["serve"]
+CMD ["server"]

README.md

@@ -3,7 +3,7 @@
 [![IC](https://github.com/go-shiori/shiori/actions/workflows/push.yml/badge.svg?branch=master)](https://github.com/go-shiori/shiori/actions/workflows/push.yml)
 [![Go Report Card](https://goreportcard.com/badge/github.com/go-shiori/shiori)](https://goreportcard.com/report/github.com/go-shiori/shiori)
 [![#[email protected]](https://img.shields.io/badge/irc-%23shiori-orange)](https://web.libera.chat/#shiori)
-[![#shiori-general:matrix.org](https://img.shields.io/badge/matrix-%23shiori-orange)](https://matrix.to/#/#shiori-general:matrix.org)
+[![#shiori-general:matrix.org](https://img.shields.io/badge/matrix-%23shiori-orange)](https://matrix.to/#/#shiori:matrix.org)
 [![Containers](https://img.shields.io/static/v1?label=Container&message=Images&color=1488C6&logo=docker)](https://github.com/go-shiori/shiori/pkgs/container/shiori)
 
 **Check out our latest [Announcements](https://github.com/go-shiori/shiori/discussions/categories/announcements)**

docs/Storage.md

@@ -0,0 +1,14 @@
+# Storage
+
+Shiori requires a folder to store several pieces of data, such as the bookmark archives, thumbnails, ebooks, and others. If the database engine used is sqlite, then the database file will also be stored in this folder.
+
+You can specify the storage folder by using `--storage-dir` or `--portable` flags when running Shiori.
+
+If none specified, Shiori will try to find the correct app folder for your OS.
+
+For example:
+- In Windows, Shiori will use `%APPDATA%`.
+- In Linux, it will use `$XDG_CONFIG_HOME` or `$HOME/.local/share` if `$XDG_CONFIG_HOME` is not set.
+- In macOS, it will use `$HOME/Library/Application Support`.
+
+> For more and up to date information about app folder discovery check [muesli/go-app-paths](https://github.com/muesli/go-app-paths)

docs/index.md

@@ -4,11 +4,12 @@ Shiori is a simple bookmarks manager written in Go language. Intended as a simpl
 
 ## Resources
 
-- [API](./API.md)
+- [API](./API.md) (Deprecated)
 - [APIv1](./APIv1.md) ([What is this?](https://github.com/go-shiori/shiori/issues/640))
 - [Contributing](./Contribute.md)
 - [Configuration](./Configuration.md)
 - [FAQ](./Frequently-Asked-Question.md)
 - [Installation](./Installation.md)
 - [Screenshots](./screenshots/)
+- [Storage](./Storage.md)
 - [Usage](./Usage.md)

internal/cmd/add.go

@@ -29,6 +29,8 @@ func addCmd() *cobra.Command {
 }
 
 func addHandler(cmd *cobra.Command, args []string) {
+	cfg, deps := initShiori(cmd.Context(), cmd)
+
 	// Read flag and arguments
 	url := args[0]
 	title, _ := cmd.Flags().GetString("title")
@@ -70,7 +72,7 @@ func addHandler(cmd *cobra.Command, args []string) {
 	}
 
 	// Save bookmark to database
-	books, err := db.SaveBookmarks(cmd.Context(), true, book)
+	books, err := deps.Database.SaveBookmarks(cmd.Context(), true, book)
 	if err != nil {
 		cError.Printf("Failed to save bookmark: %v\n", err)
 		os.Exit(1)
@@ -90,7 +92,7 @@ func addHandler(cmd *cobra.Command, args []string) {
 
 		if err == nil && content != nil {
 			request := core.ProcessRequest{
-				DataDir:     dataDir,
+				DataDir:     cfg.Storage.DataDir,
 				Bookmark:    book,
 				Content:     content,
 				ContentType: contentType,
@@ -112,7 +114,7 @@ func addHandler(cmd *cobra.Command, args []string) {
 		}
 
 		// Save bookmark to database
-		_, err = db.SaveBookmarks(cmd.Context(), false, book)
+		_, err = deps.Database.SaveBookmarks(cmd.Context(), false, book)
 		if err != nil {
 			cError.Printf("Failed to save bookmark with content: %v\n", err)
 			os.Exit(1)

internal/cmd/check.go

@@ -29,6 +29,8 @@ func checkCmd() *cobra.Command {
 }
 
 func checkHandler(cmd *cobra.Command, args []string) {
+	_, deps := initShiori(cmd.Context(), cmd)
+
 	// Parse flags
 	skipConfirm, _ := cmd.Flags().GetBool("yes")
 
@@ -53,7 +55,7 @@ func checkHandler(cmd *cobra.Command, args []string) {
 
 	// Fetch bookmarks from database
 	filterOptions := database.GetBookmarksOptions{IDs: ids}
-	bookmarks, err := db.GetBookmarks(cmd.Context(), filterOptions)
+	bookmarks, err := deps.Database.GetBookmarks(cmd.Context(), filterOptions)
 	if err != nil {
 		cError.Printf("Failed to get bookmarks: %v\n", err)
 		os.Exit(1)

internal/cmd/delete.go

@@ -29,6 +29,8 @@ func deleteCmd() *cobra.Command {
 }
 
 func deleteHandler(cmd *cobra.Command, args []string) {
+	cfg, deps := initShiori(cmd.Context(), cmd)
+
 	// Parse flags
 	skipConfirm, _ := cmd.Flags().GetBool("yes")
 
@@ -52,23 +54,23 @@ func deleteHandler(cmd *cobra.Command, args []string) {
 	}
 
 	// Delete bookmarks from database
-	err = db.DeleteBookmarks(cmd.Context(), ids...)
+	err = deps.Database.DeleteBookmarks(cmd.Context(), ids...)
 	if err != nil {
 		cError.Printf("Failed to delete bookmarks: %v\n", err)
 		os.Exit(1)
 	}
 
 	// Delete thumbnail image and archives from local disk
 	if len(ids) == 0 {
-		thumbDir := fp.Join(dataDir, "thumb")
-		archiveDir := fp.Join(dataDir, "archive")
+		thumbDir := fp.Join(cfg.Storage.DataDir, "thumb")
+		archiveDir := fp.Join(cfg.Storage.DataDir, "archive")
 		os.RemoveAll(thumbDir)
 		os.RemoveAll(archiveDir)
 	} else {
 		for _, id := range ids {
 			strID := strconv.Itoa(id)
-			imgPath := fp.Join(dataDir, "thumb", strID)
-			archivePath := fp.Join(dataDir, "archive", strID)
+			imgPath := fp.Join(cfg.Storage.DataDir, "thumb", strID)
+			archivePath := fp.Join(cfg.Storage.DataDir, "archive", strID)
 
 			os.Remove(imgPath)
 			os.Remove(archivePath)

internal/cmd/export.go

@@ -24,8 +24,10 @@ func exportCmd() *cobra.Command {
 }
 
 func exportHandler(cmd *cobra.Command, args []string) {
+	_, deps := initShiori(cmd.Context(), cmd)
+
 	// Fetch bookmarks from database
-	bookmarks, err := db.GetBookmarks(cmd.Context(), database.GetBookmarksOptions{})
+	bookmarks, err := deps.Database.GetBookmarks(cmd.Context(), database.GetBookmarksOptions{})
 	if err != nil {
 		cError.Printf("Failed to get bookmarks: %v\n", err)
 		os.Exit(1)

internal/cmd/import.go

@@ -29,6 +29,8 @@ func importCmd() *cobra.Command {
 }
 
 func importHandler(cmd *cobra.Command, args []string) {
+	_, deps := initShiori(cmd.Context(), cmd)
+
 	// Parse flags
 	generateTag := cmd.Flags().Changed("generate-tag")
 
@@ -104,7 +106,7 @@ func importHandler(cmd *cobra.Command, args []string) {
 			return
 		}
 
-		_, exist, err := db.GetBookmark(cmd.Context(), 0, url)
+		_, exist, err := deps.Database.GetBookmark(cmd.Context(), 0, url)
 		if err != nil && !errors.Is(err, sql.ErrNoRows) {
 			cError.Printf("Skip %s: Get Bookmark fail, %v", url, err)
 			return
@@ -145,7 +147,7 @@ func importHandler(cmd *cobra.Command, args []string) {
 	})
 
 	// Save bookmark to database
-	bookmarks, err = db.SaveBookmarks(cmd.Context(), true, bookmarks...)
+	bookmarks, err = deps.Database.SaveBookmarks(cmd.Context(), true, bookmarks...)
 	if err != nil {
 		cError.Printf("Failed to save bookmarks: %v\n", err)
 		os.Exit(1)

internal/cmd/open.go

@@ -36,6 +36,8 @@ func openCmd() *cobra.Command {
 }
 
 func openHandler(cmd *cobra.Command, args []string) {
+	cfg, deps := initShiori(cmd.Context(), cmd)
+
 	// Parse flags
 	skipConfirm, _ := cmd.Flags().GetBool("yes")
 	archiveMode, _ := cmd.Flags().GetBool("archive")
@@ -73,7 +75,7 @@ func openHandler(cmd *cobra.Command, args []string) {
 		WithContent: true,
 	}
 
-	bookmarks, err := db.GetBookmarks(cmd.Context(), getOptions)
+	bookmarks, err := deps.Database.GetBookmarks(cmd.Context(), getOptions)
 	if err != nil {
 		cError.Printf("Failed to get bookmarks: %v\n", err)
 		os.Exit(1)
@@ -130,7 +132,7 @@ func openHandler(cmd *cobra.Command, args []string) {
 
 	// Open archive
 	id := strconv.Itoa(bookmarks[0].ID)
-	archivePath := fp.Join(dataDir, "archive", id)
+	archivePath := fp.Join(cfg.Storage.DataDir, "archive", id)
 
 	archive, err := warc.Open(archivePath)
 	if err != nil {

internal/cmd/pocket.go

@@ -25,6 +25,8 @@ func pocketCmd() *cobra.Command {
 }
 
 func pocketHandler(cmd *cobra.Command, args []string) {
+	_, deps := initShiori(cmd.Context(), cmd)
+
 	// Open pocket's file
 	srcFile, err := os.Open(args[0])
 	if err != nil {
@@ -70,7 +72,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
 			return
 		}
 
-		_, exist, err := db.GetBookmark(cmd.Context(), 0, url)
+		_, exist, err := deps.Database.GetBookmark(cmd.Context(), 0, url)
 		if err != nil {
 			cError.Printf("Skip %s: Get Bookmark fail, %v", url, err)
 			return
@@ -103,7 +105,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
 	})
 
 	// Save bookmark to database
-	bookmarks, err = db.SaveBookmarks(cmd.Context(), true, bookmarks...)
+	bookmarks, err = deps.Database.SaveBookmarks(cmd.Context(), true, bookmarks...)
 	if err != nil {
 		cError.Printf("Failed to save bookmarks: %v\n", err)
 		os.Exit(1)

internal/cmd/print.go

@@ -32,6 +32,8 @@ func printCmd() *cobra.Command {
 }
 
 func printHandler(cmd *cobra.Command, args []string) {
+	_, deps := initShiori(cmd.Context(), cmd)
+
 	// Read flags
 	tags, _ := cmd.Flags().GetStringSlice("tags")
 	keyword, _ := cmd.Flags().GetString("search")
@@ -61,7 +63,7 @@ func printHandler(cmd *cobra.Command, args []string) {
 		OrderMethod:  orderMethod,
 	}
 
-	bookmarks, err := db.GetBookmarks(cmd.Context(), searchOptions)
+	bookmarks, err := deps.Database.GetBookmarks(cmd.Context(), searchOptions)
 	if err != nil {
 		cError.Printf("Failed to get bookmarks: %v\n", err)
 		return