Add npmQueryCacheTTL config (#921)

HOSTING.md

@@ -89,6 +89,7 @@ Available environment variables:
 - `NPM_TOKEN`: The access token for the global NPM registry.
 - `NPM_USER`: The access user for the global NPM registry.
 - `NPM_PASSWORD`: The access password for the global NPM registry.
+- `NPM_QUERY_CACHE_TTL`: The cache TTL for NPM query, default is 10 minutes.
 
 You can also create your own Dockerfile based on `ghcr.io/esm-dev/esm.sh`:
 

config.example.jsonc

@@ -47,7 +47,7 @@
   // Cache raw files in the storage, default is false.
   // The server cleans up pnpm store periodically, to avoid unnecessary `pnpm install` when accessing
   // package raw files, you can enable this option to cache the raw files in the storage.
-  // Note: this option will increase the storage usage, we recommend to enable it when you are using
+  // Note: this option will increase the storage usage, we recommend to enable it if you are using
   // a S3-compatible storage.
   "cacheRawFile": false,
 
@@ -84,6 +84,9 @@
     }
   },
 
+  // The cache TTL for npm packages query, default is 600 seconds (10 minutes).
+  "npmQueryCacheTTL": 600,
+
   // compress http response body with gzip/brotli, default is true.
   "compress": true,
 

server/cache.go

@@ -17,7 +17,7 @@ type CacheItem struct {
 
 func withCache[T any](key string, cacheTtl time.Duration, fetch func() (T, error)) (r T, err error) {
 	// check cache first
-	if cacheTtl > time.Second {
+	if cacheTtl > 0 {
 		if v, ok := cacheStore.Load(key); ok {
 			item := v.(*CacheItem)
 			if item.exp.After(time.Now()) {
@@ -33,7 +33,7 @@ func withCache[T any](key string, cacheTtl time.Duration, fetch func() (T, error
 	defer lock.(*sync.Mutex).Unlock()
 
 	// check cache again after lock
-	if cacheTtl > time.Second {
+	if cacheTtl > 0 {
 		if v, ok := cacheStore.Load(key); ok {
 			item := v.(*CacheItem)
 			if item.exp.After(time.Now()) {
@@ -47,7 +47,7 @@ func withCache[T any](key string, cacheTtl time.Duration, fetch func() (T, error
 		return
 	}
 
-	if cacheTtl > time.Second {
+	if cacheTtl > 0 {
 		cacheStore.Store(key, &CacheItem{
 			data: r,
 			exp:  time.Now().Add(cacheTtl),

server/config.go

@@ -8,11 +8,17 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/esm-dev/esm.sh/server/storage"
 )
 
+var (
+	// global config
+	config Config
+)
+
 // Config represents the configuration of esm.sh server.
 type Config struct {
 	Port             uint16                 `json:"port"`
@@ -32,6 +38,7 @@ type Config struct {
 	NpmUser          string                 `json:"npmUser"`
 	NpmPassword      string                 `json:"npmPassword"`
 	NpmRegistries    map[string]NpmRegistry `json:"npmRegistries"`
+	NpmQueryCacheTTL uint32                 `json:"npmQueryCacheTTL"`
 	MinifyRaw        json.RawMessage        `json:"minify"`
 	SourceMapRaw     json.RawMessage        `json:"sourceMap"`
 	CompressRaw      json.RawMessage        `json:"compress"`
@@ -179,6 +186,18 @@ func normalizeConfig(c *Config) {
 		}
 		c.NpmRegistries = regs
 	}
+	if c.NpmQueryCacheTTL == 0 {
+		v := os.Getenv("NPM_QUERY_CACHE_TTL")
+		if v != "" {
+			i, e := strconv.Atoi(v)
+			if e == nil && i >= 0 {
+				c.NpmQueryCacheTTL = uint32(i)
+			} else {
+				c.NpmQueryCacheTTL = 600
+			}
+		}
+		c.NpmQueryCacheTTL = 600
+	}
 	c.Compress = !(bytes.Equal(c.CompressRaw, []byte("false")) || os.Getenv("COMPRESS") == "false")
 	c.SourceMap = !(bytes.Equal(c.SourceMapRaw, []byte("false")) || (os.Getenv("SOURCEMAP") == "false" || os.Getenv("SOURCE_MAP") == "false"))
 	c.Minify = !(bytes.Equal(c.MinifyRaw, []byte("false")) || os.Getenv("MINIFY") == "false")

server/esm_router.go

@@ -5,7 +5,6 @@ import (
 	"crypto/sha1"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -21,7 +20,6 @@ import (
 	"github.com/esm-dev/esm.sh/server/common"
 	"github.com/esm-dev/esm.sh/server/storage"
 	esbuild "github.com/evanw/esbuild/pkg/api"
-	"github.com/ije/gox/crypto/rs"
 	"github.com/ije/gox/utils"
 	"github.com/ije/gox/valid"
 	"github.com/ije/rex"
@@ -30,7 +28,6 @@ import (
 
 const (
 	ccMustRevalidate = "public, max-age=0, must-revalidate"
-	cc1hour          = "public, max-age=3600"
 	cc1day           = "public, max-age=86400"
 	ccImmutable      = "public, max-age=31536000, immutable"
 	ctJavaScript     = "application/javascript; charset=utf-8"
@@ -292,16 +289,18 @@ func esmRouter(debug bool) rex.Handle {
 			buildQueue.lock.RUnlock()
 
 			disk := "ok"
-			tmpFilepath := path.Join(os.TempDir(), rs.Hex.String(32))
-			err := os.WriteFile(tmpFilepath, make([]byte, MB), 0644)
-			if err != nil {
-				if errors.Is(err, syscall.ENOSPC) {
+			var stat syscall.Statfs_t
+			err := syscall.Statfs(config.WorkDir, &stat)
+			if err == nil {
+				avail := stat.Bavail * uint64(stat.Bsize)
+				if avail < 100*MB {
 					disk = "full"
-				} else {
-					disk = "error"
+				} else if avail < 1000*MB {
+					disk = "low"
 				}
+			} else {
+				disk = "error"
 			}
-			os.Remove(tmpFilepath)
 
 			ctx.SetHeader("Cache-Control", ccMustRevalidate)
 			return map[string]any{
@@ -1016,7 +1015,7 @@ func esmRouter(debug bool) rex.Handle {
 				if rawQuery != "" {
 					query = "?" + rawQuery
 				}
-				ctx.SetHeader("Cache-Control", cc1hour)
+				ctx.SetHeader("Cache-Control", fmt.Sprintf("public, max-age=%d", config.NpmQueryCacheTTL))
 				return rex.Redirect(fmt.Sprintf("%s/%s%s%s", cdnOrigin, esm.PackageName(), subPath, query), http.StatusFound)
 			}
 			if pathKind != ESMEntry {
@@ -1040,7 +1039,7 @@ func esmRouter(debug bool) rex.Handle {
 				if rawQuery != "" {
 					qs = "?" + rawQuery
 				}
-				ctx.SetHeader("Cache-Control", cc1hour)
+				ctx.SetHeader("Cache-Control", fmt.Sprintf("public, max-age=%d", config.NpmQueryCacheTTL))
 				return rex.Redirect(fmt.Sprintf("%s%s/%s%s@%s%s%s", cdnOrigin, registryPrefix, asteriskPrefix, pkgName, pkgVersion, subPath, qs), http.StatusFound)
 			}
 		} else {
@@ -1250,7 +1249,7 @@ func esmRouter(debug bool) rex.Handle {
 			if rawQuery != "" {
 				qs = "?" + rawQuery
 			}
-			ctx.SetHeader("Cache-Control", cc1hour)
+			ctx.SetHeader("Cache-Control", fmt.Sprintf("public, max-age=%d", config.NpmQueryCacheTTL))
 			if targetFromUA {
 				appendVaryHeader(ctx.W.Header(), "User-Agent")
 			}
@@ -1630,7 +1629,7 @@ func esmRouter(debug bool) rex.Handle {
 		if isFixedVersion {
 			ctx.SetHeader("Cache-Control", ccImmutable)
 		} else {
-			ctx.SetHeader("Cache-Control", cc1hour)
+			ctx.SetHeader("Cache-Control", fmt.Sprintf("public, max-age=%d", config.NpmQueryCacheTTL))
 		}
 		ctx.SetHeader("Content-Type", ctJavaScript)
 		if ctx.R.Method == http.MethodHead {

server/git.go

@@ -26,7 +26,7 @@ type GitRef struct {
 
 // list repo refs using `git ls-remote repo`
 func listRepoRefs(repo string) (refs []GitRef, err error) {
-	return withCache(fmt.Sprintf("git ls-remote %s", repo), 10*time.Minute, func() ([]GitRef, error) {
+	return withCache(fmt.Sprintf("git ls-remote %s", repo), time.Duration(config.NpmQueryCacheTTL)*time.Second, func() ([]GitRef, error) {
 		cmd := exec.Command("git", "ls-remote", repo)
 		stdout := bytes.NewBuffer(nil)
 		errout := bytes.NewBuffer(nil)

server/npm.go

@@ -348,7 +348,7 @@ func (npmrc *NpmRC) fetchPackageInfo(packageName string, semverOrDistTag string)
 		url += "/" + semverOrDistTag
 	}
 
-	return withCache(url+"@"+semverOrDistTag+","+token+","+user+":"+password, 10*time.Minute, func() (*PackageJSON, error) {
+	return withCache(url+"@"+semverOrDistTag+","+token+","+user+":"+password, time.Duration(config.NpmQueryCacheTTL)*time.Second, func() (*PackageJSON, error) {
 		req, err := http.NewRequest("GET", url, nil)
 		if err != nil {
 			return nil, err

server/server.go

@@ -14,7 +14,6 @@ import (
 )
 
 var (
-	config       *Config
 	log          *logger.Logger
 	buildQueue   *BuildQueue
 	buildStorage storage.Storage
@@ -33,16 +32,17 @@ func Serve(efs EmbedFS) {
 	flag.Parse()
 
 	if !existsFile(cfile) {
-		config = DefaultConfig()
+		config = *DefaultConfig()
 		if cfile != "config.json" {
 			fmt.Println("Config file not found, use default config")
 		}
 	} else {
-		config, err = LoadConfig(cfile)
+		c, err := LoadConfig(cfile)
 		if err != nil {
 			fmt.Println(err.Error())
 			os.Exit(1)
 		}
+		config = *c
 		if debug {
 			fmt.Println("Config loaded from", cfile)
 		}

test/fix-url/test.ts

@@ -26,7 +26,7 @@ Deno.test("redirect semantic versioning module for deno target", async () => {
     const res = await fetch("http://localhost:8080/preact", { redirect: "manual" });
     res.body?.cancel();
     assertEquals(res.status, 302);
-    assertEquals(res.headers.get("cache-control"), "public, max-age=3600");
+    assertEquals(res.headers.get("cache-control"), "public, max-age=600");
     assertStringIncludes(res.headers.get("location")!, "http://localhost:8080/preact@");
     assertStringIncludes(res.headers.get("vary")!, "User-Agent");
   }
@@ -36,7 +36,7 @@ Deno.test("redirect semantic versioning module for deno target", async () => {
     const res = await fetch("http://localhost:8080/preact", { redirect: "manual", headers: { "User-Agent": "ES/2022" } });
     const code = await res.text();
     assertEquals(res.status, 200);
-    assertEquals(res.headers.get("cache-control"), "public, max-age=3600");
+    assertEquals(res.headers.get("cache-control"), "public, max-age=600");
     assertEquals(res.headers.get("content-type"), "application/javascript; charset=utf-8");
     assertStringIncludes(res.headers.get("vary")!, "User-Agent");
     assertStringIncludes(code, "/preact@");
@@ -48,7 +48,7 @@ Deno.test("redirect asset URLs", async () => {
   const res = await fetch("http://localhost:8080/preact/package.json", { redirect: "manual" });
   res.body?.cancel();
   assertEquals(res.status, 302);
-  assertEquals(res.headers.get("cache-control"), "public, max-age=3600");
+  assertEquals(res.headers.get("cache-control"), "public, max-age=600");
   assertStringIncludes(res.headers.get("location")!, "http://localhost:8080/preact@");
 
   const res2 = await fetch(res.headers.get("location")!, { redirect: "manual" });
@@ -60,7 +60,7 @@ Deno.test("redirect asset URLs", async () => {
   const res3 = await fetch("http://localhost:8080/preact@10/package.json", { redirect: "manual" });
   res3.body?.cancel();
   assertEquals(res3.status, 302);
-  assertEquals(res3.headers.get("cache-control"), "public, max-age=3600");
+  assertEquals(res3.headers.get("cache-control"), "public, max-age=600");
   assertStringIncludes(res3.headers.get("location")!, "http://localhost:8080/preact@10.");
 
   const res4 = await fetch(res.headers.get("location")!, { redirect: "manual" });