From dc652816ad904b7a265fbdd3b51f8a43037fb22c Mon Sep 17 00:00:00 2001 From: simulot Date: Fri, 31 May 2024 19:07:43 +0200 Subject: [PATCH 1/9] Fix: report the xmp file name in the log --- browser/files/localassets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser/files/localassets.go b/browser/files/localassets.go index 2150dcd4..52fae218 100644 --- a/browser/files/localassets.go +++ b/browser/files/localassets.go @@ -182,7 +182,7 @@ func (la *LocalAssetBrowser) checkSidecar(ctx context.Context, f *browser.LocalA FileName: path.Join(dir, e.Name()), OnFSsys: true, } - la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, f.FileName, "main", f.FileName) + la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, path.Join(dir, e.Name()), "main", f.FileName) return true } } From 1bc587c20c6ebb80430e84a42054ad11a2b700bc Mon Sep 17 00:00:00 2001 From: simulot Date: Fri, 31 May 2024 21:48:40 +0200 Subject: [PATCH 2/9] better log handler initialisation --- cmd/shared.go | 9 +++++++-- cmd/upload/page.go | 5 ++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/shared.go b/cmd/shared.go index 59919b66..d84a537d 100644 --- a/cmd/shared.go +++ b/cmd/shared.go @@ -100,8 +100,7 @@ func (app *SharedFlags) Start(ctx context.Context) error { if err != nil { return err } - app.Log = slog.New(humane.NewHandler(f, &humane.Options{Level: app.Level})) - app.Jnl.SetLogger(app.Log) + app.SetLogWriter(f) app.LogWriterCloser = f } } @@ -176,3 +175,9 @@ func (app *SharedFlags) Start(ctx context.Context) error { } return nil } + +func (app *SharedFlags) SetLogWriter(w io.Writer) { + app.Log = slog.New(humane.NewHandler(w, &humane.Options{Level: app.Level})) + // app.Log = slog.New(slog.NewJSONHandler(w, &slog.HandlerOptions{})) + app.Jnl.SetLogger(app.Log) +} diff --git a/cmd/upload/page.go b/cmd/upload/page.go index a1b76623..e524fc75 100644 --- a/cmd/upload/page.go +++ b/cmd/upload/page.go @@ -10,7 +10,6 @@ import ( "github.com/navidys/tvxwidgets" "github.com/rivo/tview" "github.com/simulot/immich-go/helpers/fileevent" - "github.com/telemachus/humane" ) type page struct { @@ -88,9 +87,9 @@ func (p *page) Page() *tview.Application { if p.app.SharedFlags.LogWriterCloser != nil { w := io.MultiWriter(p.app.SharedFlags.LogWriterCloser, p.logView) - p.app.SharedFlags.Log = slog.New(humane.NewHandler(w, &humane.Options{Level: p.app.SharedFlags.Level})) + p.app.SetLogWriter(w) } else { - p.app.SharedFlags.Log = slog.New(humane.NewHandler(p.logView, &humane.Options{Level: p.app.SharedFlags.Level})) + p.app.SetLogWriter(p.logView) } p.app.SharedFlags.Jnl.SetLogger(p.app.SharedFlags.Log) p.logView.SetBorder(true).SetTitle("Log") From 17b0caf6271fd8513203235f4006b4055d92bafc Mon Sep 17 00:00:00 2001 From: simulot Date: Fri, 31 May 2024 21:49:44 +0200 Subject: [PATCH 3/9] fix: usupported files are counted as discared files --- browser/files/localassets.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/browser/files/localassets.go b/browser/files/localassets.go index 52fae218..9eb803fb 100644 --- a/browser/files/localassets.go +++ b/browser/files/localassets.go @@ -107,10 +107,10 @@ nextFile: t := la.sm.TypeFromExt(ext) switch t { default: - la.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, fileName, "reason", "unsupported file type") + la.log.Record(ctx, fileevent.DiscoveredUnsupported, nil, fileName, "reason", "unsupported file type") continue nextFile case immich.TypeIgnored: - la.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, fileName, "reason", "useless file") + la.log.Record(ctx, fileevent.DiscoveredUnsupported, nil, fileName, "reason", "useless file") continue nextFile case immich.TypeSidecar: la.log.Record(ctx, fileevent.DiscoveredSidecar, nil, fileName) From 7e5bede978c4a78c23f73bab780bc8e9bea53b28 Mon Sep 17 00:00:00 2001 From: simulot Date: Sat, 1 Jun 2024 09:27:33 +0200 Subject: [PATCH 4/9] fix the immich reading progression bar --- cmd/upload/upload.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/upload/upload.go b/cmd/upload/upload.go index 3070dc12..c9c963ea 100644 --- a/cmd/upload/upload.go +++ b/cmd/upload/upload.go @@ -382,6 +382,9 @@ func (app *UpCmd) getImmichAssets(ctx context.Context, updateFn progressUpdate) if err != nil { return err } + if updateFn != nil { + updateFn(totalOnImmich, totalOnImmich) + } app.AssetIndex = &AssetIndex{ assets: list, } From 26204a93abaa262985a78664f2bd80eff1bc61eb Mon Sep 17 00:00:00 2001 From: simulot Date: Sat, 1 Jun 2024 10:47:12 +0200 Subject: [PATCH 5/9] show procecessed in no-gui mode --- cmd/upload/upload.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/upload/upload.go b/cmd/upload/upload.go index c9c963ea..d7fd9b4d 100644 --- a/cmd/upload/upload.go +++ b/cmd/upload/upload.go @@ -245,22 +245,28 @@ func (app *UpCmd) runNoUI(ctx context.Context) error { if maxImmich > 0 { immichPct = 100 * currImmich / maxImmich } + ScannedAssets := counts[fileevent.DiscoveredImage] + counts[fileevent.DiscoveredVideo] + ProcessedAssets := counts[fileevent.UploadNotSelected] + + counts[fileevent.UploadUpgraded] + + counts[fileevent.UploadServerDuplicate] + + counts[fileevent.UploadServerBetter] + + counts[fileevent.Uploaded] + + counts[fileevent.AnalysisLocalDuplicate] + counts[fileevent.AnalysisMissingAssociatedMetadata] if app.GooglePhotos { gpPct := 0 upPct := 0 - if counts[fileevent.DiscoveredImage]+counts[fileevent.DiscoveredVideo] > 0 { - gpPct = int(100 * counts[fileevent.AnalysisAssociatedMetadata] / (counts[fileevent.DiscoveredImage] + counts[fileevent.DiscoveredVideo])) + if ScannedAssets > 0 { + gpPct = int(100 * counts[fileevent.AnalysisAssociatedMetadata] / ScannedAssets) upPct = int(100 * (counts[fileevent.UploadNotSelected] + counts[fileevent.UploadUpgraded] + counts[fileevent.UploadServerDuplicate] + counts[fileevent.UploadServerBetter] + counts[fileevent.Uploaded] + - counts[fileevent.AnalysisLocalDuplicate]) / - (counts[fileevent.DiscoveredImage] + counts[fileevent.DiscoveredVideo])) + counts[fileevent.AnalysisLocalDuplicate]) / ScannedAssets) } s = fmt.Sprintf("\rImmich read %d%%, Google Photos Analysis: %d%%, Uploaded %d%% %s", immichPct, gpPct, upPct, string(spinner[spinIdx])) } else { - s = fmt.Sprintf("\rImmich read %d%%, Uploaded %d %s", immichPct, counts[fileevent.Uploaded], string(spinner[spinIdx])) + s = fmt.Sprintf("\rImmich read %d%%, Processed %d, Uploaded %d %s", immichPct, ProcessedAssets, counts[fileevent.Uploaded], string(spinner[spinIdx])) } spinIdx++ if spinIdx == len(spinner) { @@ -652,8 +658,8 @@ func (app *UpCmd) handleAsset(ctx context.Context, a *browser.LocalAssetFile) er if err != nil { app.Jnl.Record(ctx, fileevent.UploadServerError, a, a.FileName, "error", err.Error()) } + time.Sleep(2 * time.Millisecond) } - time.Sleep(2 * time.Millisecond) return nil } From 31b32c3afc9fa20038cc5622e4e65658095084a3 Mon Sep 17 00:00:00 2001 From: simulot Date: Sat, 1 Jun 2024 10:47:46 +0200 Subject: [PATCH 6/9] write report --- helpers/fileevent/fileevents.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/helpers/fileevent/fileevents.go b/helpers/fileevent/fileevents.go index 30892625..811ceb63 100644 --- a/helpers/fileevent/fileevents.go +++ b/helpers/fileevent/fileevents.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "strings" "sync" "sync/atomic" ) @@ -124,9 +125,11 @@ func (r *Recorder) SetLogger(l *slog.Logger) { } func (r *Recorder) Report() { - r.log.Info("\nInput analysis:\n----------------------") - fmt.Println("\nInput analysis:") - fmt.Println("----------------------") + sb := strings.Builder{} + + sb.WriteString("\n") + sb.WriteString("Input analysis:\n") + sb.WriteString("---------------\n") for _, c := range []Code{ DiscoveredImage, DiscoveredVideo, @@ -137,12 +140,12 @@ func (r *Recorder) Report() { AnalysisAssociatedMetadata, AnalysisMissingAssociatedMetadata, } { - r.log.Info(fmt.Sprintf("%-40s: %7d", c.String(), r.counts[c])) - fmt.Printf("%-40s: %7d\n", c.String(), r.counts[c]) + sb.WriteString(fmt.Sprintf("%-40s: %7d\n", c.String(), r.counts[c])) } - r.log.Info("\nUploading:\n----------") - fmt.Println("\nUploading:") - fmt.Println("----------") + + sb.WriteString("\n") + sb.WriteString("Uploading:\n") + sb.WriteString("----------\n") for _, c := range []Code{ Uploaded, UploadServerError, @@ -151,9 +154,11 @@ func (r *Recorder) Report() { UploadServerDuplicate, UploadServerBetter, } { - r.log.Info(fmt.Sprintf("%-40s: %7d", c.String(), r.counts[c])) - fmt.Printf("%-40s: %7d\n", c.String(), r.counts[c]) + sb.WriteString(fmt.Sprintf("%-40s: %7d\n", c.String(), r.counts[c])) } + + r.log.Info(sb.String()) + fmt.Println(sb.String()) } func (r *Recorder) GetCounts() []int64 { From b58becb27e8bb8b7966dc098c6823ecf029c4e53 Mon Sep 17 00:00:00 2001 From: simulot Date: Sat, 1 Jun 2024 11:07:16 +0200 Subject: [PATCH 7/9] add the possibility to write a JSON log --- cmd/shared.go | 10 ++++++++-- readme.md | 27 ++++++++++++++------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/cmd/shared.go b/cmd/shared.go index d84a537d..e3cacf68 100644 --- a/cmd/shared.go +++ b/cmd/shared.go @@ -35,6 +35,7 @@ type SharedFlags struct { TimeZone string // Override default TZ SkipSSL bool // Skip SSL Verification NoUI bool // Disable user interface + JSONLog bool // Enable JSON structured log Immich immich.ImmichInterface // Immich client Log *slog.Logger // Logger @@ -53,6 +54,7 @@ func (app *SharedFlags) InitSharedFlags() { app.SkipSSL = false app.LogLevel = "INFO" app.NoUI = false + app.JSONLog = false } // SetFlag add common flags to a flagset @@ -65,6 +67,7 @@ func (app *SharedFlags) SetFlags(fs *flag.FlagSet) { fs.BoolFunc("no-colors-log", "Disable colors on logs", myflag.BoolFlagFn(&app.NoLogColors, app.NoLogColors)) fs.StringVar(&app.LogLevel, "log-level", app.LogLevel, "Log level (DEBUG|INFO|WARN|ERROR), default INFO") fs.StringVar(&app.LogFile, "log-file", app.LogFile, "Write log messages into the file") + fs.BoolFunc("log-json", "Output line-delimited JSON file, default FALSE", myflag.BoolFlagFn(&app.JSONLog, app.JSONLog)) fs.BoolFunc("api-trace", "enable api call traces", myflag.BoolFlagFn(&app.APITrace, app.APITrace)) fs.BoolFunc("debug", "enable debug messages", myflag.BoolFlagFn(&app.Debug, app.Debug)) fs.StringVar(&app.TimeZone, "time-zone", app.TimeZone, "Override the system time zone") @@ -177,7 +180,10 @@ func (app *SharedFlags) Start(ctx context.Context) error { } func (app *SharedFlags) SetLogWriter(w io.Writer) { - app.Log = slog.New(humane.NewHandler(w, &humane.Options{Level: app.Level})) - // app.Log = slog.New(slog.NewJSONHandler(w, &slog.HandlerOptions{})) + if app.JSONLog { + app.Log = slog.New(slog.NewJSONHandler(w, &slog.HandlerOptions{})) + } else { + app.Log = slog.New(humane.NewHandler(w, &humane.Options{Level: app.Level})) + } app.Jnl.SetLogger(app.Log) } diff --git a/readme.md b/readme.md index 60e4acf5..6a18b0c9 100644 --- a/readme.md +++ b/readme.md @@ -57,19 +57,20 @@ immich-go -server URL -key KEY -general_options COMMAND -command_options... {fil -| **Parameter** | **Description** | **Default value** | -| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | -| `-use-configuration=path/to/config/file` | Specifies the configuration file to use.
Server URL and the API key are stored into the immich-go configuration file. They can be omitted for the next runs. | Linux `$HOME/.config/immich-go/immich-go.json`
Windows `%AppData%\immich-go\immich-go.json`
Apple `$HOME/Library/Application Support/immich-go/immich-go.json` | -| `-server=URL` | URL of the Immich service, example http://:2283 or https://your-domain | | -| `-api=URL` | URL of the Immich api endpoint (http://container_ip:3301) | | -| `-device-uuid=VALUE` | Force the device identification | `$HOSTNAME` | -| `-skip-verify-ssl` | Skip SSL verification for use with self-signed certificates | `false` | -| `-key=KEY` | A key generated by the user. Uploaded photos will belong to the key's owner. | | -| `-no-colors-log` | Remove color codes from logs. | `TRUE` on Windows, `FALSE` otherwise | -| `-log-level=LEVEL` | Adjust the log verbosity as follows:
- `ERROR`: Display only errors
- `WARNING`: Same as previous one plus non blocking error
- `INFO`: Information messages | `INFO` | -| `-log-file=/path/to/log/file` | Write all messages to a file | Linux `$HOME/.cache/immich-go/immich-go_YYYY-MM-DD_HH-MI-SS.log`
Windows `%LocalAppData%\immich-go\immich-go_YYYY-MM-DD_HH-MI-SS.log`
Apple `$HOME/Library/Caches/immich-go/immich-go_YYYY-MM-DD_HH-MI-SS.log`| -| `-time-zone=time_zone_name` | Set the time zone for dates without time zone information | the system's time zone | -| `-no-ui` | Disable the user interface | 'false' | +| **Parameter** | **Description** | **Default value** | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `-use-configuration=path/to/config/file` | Specifies the configuration file to use.
Server URL and the API key are stored into the immich-go configuration file. They can be omitted for the next runs. | Linux `$HOME/.config/immich-go/immich-go.json`
Windows `%AppData%\immich-go\immich-go.json`
Apple `$HOME/Library/Application Support/immich-go/immich-go.json` | +| `-server=URL` | URL of the Immich service, example http://:2283 or https://your-domain | | +| `-api=URL` | URL of the Immich api endpoint (http://container_ip:3301) | | +| `-device-uuid=VALUE` | Force the device identification | `$HOSTNAME` | +| `-skip-verify-ssl` | Skip SSL verification for use with self-signed certificates | `false` | +| `-key=KEY` | A key generated by the user. Uploaded photos will belong to the key's owner. | | +| `-no-colors-log` | Remove color codes from logs. | `TRUE` on Windows, `FALSE` otherwise | +| `-log-level=LEVEL` | Adjust the log verbosity as follows:
- `ERROR`: Display only errors
- `WARNING`: Same as previous one plus non blocking error
- `INFO`: Information messages | `INFO` | +| `-log-file=/path/to/log/file` | Write all messages to a file | Linux `$HOME/.cache/immich-go/immich-go_YYYY-MM-DD_HH-MI-SS.log`
Windows `%LocalAppData%\immich-go\immich-go_YYYY-MM-DD_HH-MI-SS.log`
Apple `$HOME/Library/Caches/immich-go/immich-go_YYYY-MM-DD_HH-MI-SS.log` | +| `-log-json` | Output the log as line-delimited JSON file | `false` | +| `-time-zone=time_zone_name` | Set the time zone for dates without time zone information | the system's time zone | +| `-no-ui` | Disable the user interface | 'false' | ## Command `upload` From 8b0b2aaf41e7a2d484cc087236f0a284c38d6cd6 Mon Sep 17 00:00:00 2001 From: simulot Date: Sat, 1 Jun 2024 11:07:53 +0200 Subject: [PATCH 8/9] change the wording "server has a better asset" instead of "server has a better photo" --- helpers/fileevent/fileevents.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/fileevent/fileevents.go b/helpers/fileevent/fileevents.go index 811ceb63..27ce1ff3 100644 --- a/helpers/fileevent/fileevents.go +++ b/helpers/fileevent/fileevents.go @@ -59,7 +59,7 @@ var _code = map[Code]string{ UploadNotSelected: "file not selected", UploadUpgraded: "server's asset upgraded with the input", UploadAddToAlbum: "added to an album", - UploadServerDuplicate: "server has same photo", + UploadServerDuplicate: "server has same asset", UploadServerBetter: "server has a better asset", UploadAlbumCreated: "album created/updated", UploadServerError: "server error", From 088a117ff3c896774ff8d09e8afd735f92255b37 Mon Sep 17 00:00:00 2001 From: simulot Date: Sat, 1 Jun 2024 22:02:26 +0200 Subject: [PATCH 9/9] report upload error --- cmd/upload/upload.go | 5 +++-- helpers/fileevent/fileevents.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/upload/upload.go b/cmd/upload/upload.go index d7fd9b4d..e9bf40f4 100644 --- a/cmd/upload/upload.go +++ b/cmd/upload/upload.go @@ -264,9 +264,9 @@ func (app *UpCmd) runNoUI(ctx context.Context) error { counts[fileevent.Uploaded] + counts[fileevent.AnalysisLocalDuplicate]) / ScannedAssets) } - s = fmt.Sprintf("\rImmich read %d%%, Google Photos Analysis: %d%%, Uploaded %d%% %s", immichPct, gpPct, upPct, string(spinner[spinIdx])) + s = fmt.Sprintf("\rImmich read %d%%, Google Photos Analysis: %d%%, Upload errors: %d, Uploaded %d%% %s", immichPct, gpPct, counts[fileevent.UploadServerError], upPct, string(spinner[spinIdx])) } else { - s = fmt.Sprintf("\rImmich read %d%%, Processed %d, Uploaded %d %s", immichPct, ProcessedAssets, counts[fileevent.Uploaded], string(spinner[spinIdx])) + s = fmt.Sprintf("\rImmich read %d%%, Processed %d, Upload errors: %d, Uploaded %d %s", immichPct, ProcessedAssets, counts[fileevent.UploadServerError], counts[fileevent.Uploaded], string(spinner[spinIdx])) } spinIdx++ if spinIdx == len(spinner) { @@ -605,6 +605,7 @@ func (app *UpCmd) handleAsset(ctx context.Context, a *browser.LocalAssetFile) er } if err != nil { + app.Jnl.Record(ctx, fileevent.UploadServerError, a, a.FileName, "error", err.Error()) return nil } diff --git a/helpers/fileevent/fileevents.go b/helpers/fileevent/fileevents.go index 27ce1ff3..bb6d7a77 100644 --- a/helpers/fileevent/fileevents.go +++ b/helpers/fileevent/fileevents.go @@ -62,7 +62,7 @@ var _code = map[Code]string{ UploadServerDuplicate: "server has same asset", UploadServerBetter: "server has a better asset", UploadAlbumCreated: "album created/updated", - UploadServerError: "server error", + UploadServerError: "upload error", Uploaded: "uploaded", Error: "error",