From e10e6cfe4dbda9bddba82385ceaa53d634bae7bd Mon Sep 17 00:00:00 2001 From: yusing Date: Mon, 13 Jan 2025 02:21:52 +0800 Subject: [PATCH] updated `ls-icon` and icon fetching mechanism --- internal/api/v1/favicon/favicon.go | 53 ++++++++++----- internal/homepage/icon_url.go | 36 ++++++++-- internal/homepage/icon_url_test.go | 67 ++++++++++++++++++ internal/list-icons.go | 106 ++++++++++++++++------------- schema/providers.schema.json | 2 +- 5 files changed, 191 insertions(+), 73 deletions(-) create mode 100644 internal/homepage/icon_url_test.go diff --git a/internal/api/v1/favicon/favicon.go b/internal/api/v1/favicon/favicon.go index 31a37d3b..6438a32f 100644 --- a/internal/api/v1/favicon/favicon.go +++ b/internal/api/v1/favicon/favicon.go @@ -16,6 +16,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/vincent-petithory/dataurl" + "github.com/yusing/go-proxy/internal" U "github.com/yusing/go-proxy/internal/api/v1/utils" "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/logging" @@ -78,12 +79,15 @@ func GetFavIcon(w http.ResponseWriter, req *http.Request) { var status int var errMsg string - homepage := r.RawEntry().Homepage - if homepage != nil && homepage.Icon != nil { - if homepage.Icon.IsRelative { - icon, status, errMsg = findIcon(r, req, homepage.Icon.Value) - } else { - icon, status, errMsg = getIconAbsolute(homepage.Icon.Value) + hp := r.RawEntry().Homepage + if hp != nil && hp.Icon != nil { + switch hp.Icon.IconSource { + case homepage.IconSourceAbsolute: + icon, status, errMsg = fetchIconAbsolute(hp.Icon.Value) + case homepage.IconSourceRelative: + icon, status, errMsg = findIcon(r, req, hp.Icon.Value) + case homepage.IconSourceWalkXCode: + icon, status, errMsg = fetchWalkxcodeIcon(hp.Icon.Extra.FileType, hp.Icon.Extra.Name) } } else { // try extract from "link[rel=icon]" @@ -124,7 +128,7 @@ func storeIconCache(key string, icon []byte) { iconCache[key] = icon } -func getIconAbsolute(url string) ([]byte, int, string) { +func fetchIconAbsolute(url string) ([]byte, int, string) { icon, ok := loadIconCache(url) if ok { return icon, http.StatusOK, "" @@ -165,6 +169,25 @@ func sanitizeName(name string) string { return strings.ToLower(nameSanitizer.Replace(name)) } +func fetchWalkxcodeIcon(filetype string, name string) ([]byte, int, string) { + // if icon isn't in the list, no need to fetch + if !internal.HasIcon(name, filetype) { + logging.Debug(). + Str("filetype", filetype). + Str("name", name). + Msg("icon not found") + return nil, http.StatusNotFound, "icon not found" + } + + icon, ok := loadIconCache("walkxcode/" + filetype + "/" + name) + if ok { + return icon, http.StatusOK, "" + } + + url := homepage.DashboardIconBaseURL + filetype + "/" + name + "." + filetype + return fetchIconAbsolute(url) +} + func findIcon(r route.HTTPRoute, req *http.Request, uri string) (icon []byte, status int, errMsg string) { key := r.TargetName() icon, ok := loadIconCache(key) @@ -175,10 +198,10 @@ func findIcon(r route.HTTPRoute, req *http.Request, uri string) (icon []byte, st return icon, http.StatusOK, "" } - icon, status, errMsg = getIconAbsolute(homepage.DashboardIconBaseURL + "png/" + sanitizeName(r.TargetName()) + ".png") + icon, status, errMsg = fetchWalkxcodeIcon("png", sanitizeName(r.TargetName())) cont := r.RawEntry().Container if icon == nil && cont != nil { - icon, status, errMsg = getIconAbsolute(homepage.DashboardIconBaseURL + "png/" + sanitizeName(cont.ImageName) + ".png") + icon, status, errMsg = fetchWalkxcodeIcon("png", sanitizeName(cont.ImageName)) } if icon == nil { // fallback to parse html @@ -224,10 +247,6 @@ func findIconSlow(r route.HTTPRoute, req *http.Request, uri string) (icon []byte if loc == newReq.URL.Path { return nil, http.StatusBadGateway, "circular redirect" } - logging.Debug().Str("route", r.TargetName()). - Str("from", uri). - Str("to", loc). - Msg("favicon redirect") return findIconSlow(r, req, loc) } } @@ -264,8 +283,10 @@ func findIconSlow(r route.HTTPRoute, req *http.Request, uri string) (icon []byte } return dataURI.Data, http.StatusOK, "" } - if href[0] != '/' { - return getIconAbsolute(href) + switch { + case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"): + return fetchIconAbsolute(href) + default: + return findIconSlow(r, req, path.Clean(href)) } - return findIconSlow(r, req, href) } diff --git a/internal/homepage/icon_url.go b/internal/homepage/icon_url.go index 471abcfa..c87a8460 100644 --- a/internal/homepage/icon_url.go +++ b/internal/homepage/icon_url.go @@ -6,10 +6,26 @@ import ( E "github.com/yusing/go-proxy/internal/error" ) -type IconURL struct { - Value string `json:"value"` - IsRelative bool `json:"is_relative"` -} +type ( + IconURL struct { + Value string `json:"value"` + IconSource + Extra *IconExtra `json:"extra"` + } + + IconExtra struct { + FileType string `json:"file_type"` + Name string `json:"name"` + } + + IconSource int +) + +const ( + IconSourceAbsolute IconSource = iota + IconSourceRelative + IconSourceWalkXCode +) const DashboardIconBaseURL = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/" @@ -28,13 +44,19 @@ func (u *IconURL) Parse(v string) error { switch beforeSlash { case "http:", "https:": u.Value = v + u.IconSource = IconSourceAbsolute return nil case "@target": u.Value = v[slashIndex:] - u.IsRelative = true + u.IconSource = IconSourceRelative return nil - case "png", "svg": // walkXCode Icons - u.Value = DashboardIconBaseURL + v + case "png", "svg", "webp": // walkXCode Icons + u.Value = v + u.IconSource = IconSourceWalkXCode + u.Extra = &IconExtra{ + FileType: beforeSlash, + Name: strings.TrimSuffix(v[slashIndex+1:], "."+beforeSlash), + } return nil default: return ErrInvalidIconURL diff --git a/internal/homepage/icon_url_test.go b/internal/homepage/icon_url_test.go new file mode 100644 index 00000000..89f53b2b --- /dev/null +++ b/internal/homepage/icon_url_test.go @@ -0,0 +1,67 @@ +package homepage + +import ( + "testing" + + . "github.com/yusing/go-proxy/internal/utils/testing" +) + +func TestIconURL(t *testing.T) { + tests := []struct { + name string + input string + wantValue *IconURL + wantErr bool + }{ + { + name: "absolute", + input: "http://example.com/icon.png", + wantValue: &IconURL{ + Value: "http://example.com/icon.png", + IconSource: IconSourceAbsolute, + }, + }, + { + name: "relative", + input: "@target/icon.png", + wantValue: &IconURL{ + Value: "/icon.png", + IconSource: IconSourceRelative, + }, + }, + { + name: "walkxcode", + input: "png/walkxcode.png", + wantValue: &IconURL{ + Value: "png/walkxcode.png", + IconSource: IconSourceWalkXCode, + Extra: &IconExtra{ + FileType: "png", + Name: "walkxcode", + }, + }, + }, + { + name: "invalid", + input: "invalid", + wantErr: true, + }, + { + name: "empty", + input: "", + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + u := &IconURL{} + err := u.Parse(tc.input) + if tc.wantErr { + ExpectError(t, ErrInvalidIconURL, err) + } else { + ExpectNoError(t, err) + ExpectDeepEqual(t, u, tc.wantValue) + } + }) + } +} diff --git a/internal/list-icons.go b/internal/list-icons.go index 8aff4bcb..3f9f0a54 100644 --- a/internal/list-icons.go +++ b/internal/list-icons.go @@ -2,14 +2,12 @@ package internal import ( "encoding/json" - "fmt" "io" - "log" "net/http" - "os" + "sync" "time" - "github.com/yusing/go-proxy/internal/utils" + "github.com/yusing/go-proxy/internal/logging" ) type GitHubContents struct { //! keep this, may reuse in future @@ -20,54 +18,71 @@ type GitHubContents struct { //! keep this, may reuse in future Size int `json:"size"` } -const ( - iconsCachePath = "/tmp/icons_cache.json" - updateInterval = 1 * time.Hour +type Icons map[string]map[string]struct{} + +// no longer cache for `godoxy ls-icons` + +const updateInterval = 1 * time.Hour + +var ( + iconsCache = make(Icons) + iconsCahceMu sync.Mutex + lastUpdate time.Time ) -func ListAvailableIcons() ([]string, error) { - owner := "walkxcode" - repo := "dashboard-icons" - ref := "main" +const walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json" - var lastUpdate time.Time +func ListAvailableIcons() (Icons, error) { + iconsCahceMu.Lock() + defer iconsCahceMu.Unlock() - icons := make([]string, 0) - info, err := os.Stat(iconsCachePath) - if err == nil { - lastUpdate = info.ModTime().Local() - } if time.Since(lastUpdate) < updateInterval { - err := utils.LoadJSON(iconsCachePath, &icons) - if err == nil { - return icons, nil + if len(iconsCache) > 0 { + return iconsCache, nil } } - contents, err := getRepoContents(http.DefaultClient, owner, repo, ref, "") + icons, err := getIcons() if err != nil { return nil, err } - for _, content := range contents { - if content.Type != "dir" { - icons = append(icons, content.Path) - } - } - err = utils.SaveJSON(iconsCachePath, &icons, 0o644) + + iconsCache = icons + lastUpdate = time.Now() + return icons, nil +} + +func HasIcon(name string, filetype string) bool { + icons, err := ListAvailableIcons() if err != nil { - log.Print("error saving cache", err) + logging.Error().Err(err).Msg("failed to list icons") + return false } - return icons, nil + if _, ok := icons[filetype]; !ok { + return false + } + _, ok := icons[filetype][name+"."+filetype] + return ok } -func getRepoContents(client *http.Client, owner string, repo string, ref string, path string) ([]GitHubContents, error) { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s", owner, repo, path, ref), nil) +/* +format: + + { + "png": [ + "*.png", + ], + "svg": [ + "*.svg", + ] + } +*/ +func getIcons() (Icons, error) { + req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil) if err != nil { return nil, err } - req.Header.Set("Accept", "application/json") - - resp, err := client.Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -78,24 +93,17 @@ func getRepoContents(client *http.Client, owner string, repo string, ref string, return nil, err } - var contents []GitHubContents - err = json.Unmarshal(body, &contents) + data := make(map[string][]string) + err = json.Unmarshal(body, &data) if err != nil { return nil, err } - - filesAndDirs := make([]GitHubContents, 0) - for _, content := range contents { - if content.Type == "dir" { - subContents, err := getRepoContents(client, owner, repo, ref, content.Path) - if err != nil { - return nil, err - } - filesAndDirs = append(filesAndDirs, subContents...) - } else { - filesAndDirs = append(filesAndDirs, content) + icons := make(Icons, len(data)) + for fileType, files := range data { + icons[fileType] = make(map[string]struct{}, len(files)) + for _, icon := range files { + icons[fileType][icon] = struct{}{} } } - - return filesAndDirs, nil + return icons, nil } diff --git a/schema/providers.schema.json b/schema/providers.schema.json index 96aba37e..bf123ab8 100644 --- a/schema/providers.schema.json +++ b/schema/providers.schema.json @@ -85,7 +85,7 @@ "type": "string", "oneOf": [ { - "pattern": "^(png|svg)\\/[\\w\\d\\-_]+\\.\\1$", + "pattern": "^(png|svg|webp)\\/[\\w\\d\\-_]+\\.\\1$", "title": "Icon from walkxcode/dashboard-icons" }, {