feat(archive): refine max archive workers mechanism

Author: marjune163

README.md

@@ -258,16 +258,17 @@ ghfs [options]
 -A|--global-archive
     Allow user to download the whole contents of current directory for all url paths.
     A download link will appear on top part of the page.
---archive-max-workers <number>
-    Maximum number of concurrent archive operations.
-    Set to 0 for unlimited (default).
-    When the limit is reached, new archive requests will receive 429 Too Many Requests.
+    Make sure there is no circular symbol links.
 --archive <url-path> ...
 --archive-user <separator><url-path>[<separator><allowed-username>...] ...
     Allow user to download the whole contents of current directory for specific url paths(and sub paths).
 --archive-dir <fs-path> ...
 --archive-dir-user <separator><fs-path>[<separator><allowed-username>...] ...
     Similar to --archive, but use file system path instead of url path.
+--archive-workers-max <number>
+    Maximum number of concurrent archive operations.
+    Set to 0 for unlimited (default).
+    When the limit is reached, new archive requests will be rejected with status code 429(Too Many Requests).
 
 --global-cors
     Allow CORS requests for all url path.

README.zh-CN.md

@@ -28,7 +28,7 @@ go build main.go
 在8080端口启动服务器，根目录为当前工作目录：
 ```sh
 ghfs -l 8080
-``` 
+```
 
 在8080端口启动服务器，根目录为 /usr/share/doc：
 ```sh
@@ -255,6 +255,10 @@ ghfs [选项]
 --archive-dir <文件系统路径> ...
 --archive-dir-user <分隔符><文件系统路径>[<分隔符><允许的用户名>...] ...
     与--archive类似，但指定的是文件系统路径，而不是URL路径。
+--archive-workers-max <数值>
+    指定打包下载的最大并发数。
+    设为0（默认）表示无限制。
+    达到并发上限后，会拒绝后续打包请求并返回状态码429（Too Many Requests）。
 
 --global-cors
     接受所有URL路径的CORS跨域请求。

src/param/cli.go

@@ -148,7 +148,7 @@ func NewCliCmd() *goNixArgParser.Command {
 	err = options.AddFlagValues("archivedirsusers", "--archive-dir-user", "", nil, "file system path that allow archive files for specific users, <sep><fs-path>[<sep><user>...]")
 	serverError.CheckFatal(err)
 
-	err = options.AddFlagValue("archivemaxworkers", "--archive-max-workers", "", "0", "maximum number of concurrent archive operations (0 for unlimited)")
+	err = options.AddFlagValue("archiveworkersmax", "--archive-workers-max", "", "0", "maximum number of concurrent archive operations (0 for unlimited)")
 	serverError.CheckFatal(err)
 
 	err = options.AddFlag("globalcors", "--global-cors", "GHFS_GLOBAL_CORS", "enable CORS headers for all directories")
@@ -440,7 +440,7 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e
 		archiveDirsUsers, _ := result.GetStrings("archivedirsusers")
 		param.ArchiveDirsUsers = SplitAllKeyValues(archiveDirsUsers)
 
-		param.ArchiveMaxWorkers, _ = result.GetUint32("archivemaxworkers")
+		param.ArchiveWorkersMax, _ = result.GetUint32("archiveworkersmax")
 
 		// global restrict access
 		if result.HasKey("globalrestrictaccess") {

src/param/main.go

@@ -62,7 +62,7 @@ type Param struct {
 	ArchiveUrlsUsers  [][]string // [][path, user...]
 	ArchiveDirs       []string
 	ArchiveDirsUsers  [][]string // [][path, user...]
-	ArchiveMaxWorkers uint32
+	ArchiveWorkersMax uint32
 
 	GlobalCors bool
 	CorsUrls   []string

src/serverHandler/aliasHandler.go

@@ -1,18 +1,16 @@
 package serverHandler
 
 import (
-	"net/http"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync/atomic"
-
 	"mjpclab.dev/ghfs/src/middleware"
 	"mjpclab.dev/ghfs/src/param"
 	"mjpclab.dev/ghfs/src/serverLog"
 	"mjpclab.dev/ghfs/src/tpl/theme"
 	"mjpclab.dev/ghfs/src/user"
 	"mjpclab.dev/ghfs/src/util"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
 )
 
 var defaultHandler = http.NotFoundHandler()
@@ -50,8 +48,8 @@ type aliasHandler struct {
 	archive *hierarchyAvailability
 	cors    *hierarchyAvailability
 
-	archiveMaxWorkers uint32
-	archiveWorkers    *atomic.Uint32
+	archiveWorkersMax uint32
+	archivingWorkers  *uint32
 
 	globalRestrictAccess []string
 	restrictAccessUrls   pathStringsList
@@ -125,7 +123,7 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 		if session.isMutate && h.mutate(w, r, session, data) {
 			return
-		} else if session.isArchive && h.createArchive(w, r, session, data) {
+		} else if session.isArchive && h.tryArchive(w, r, session, data) {
 			return
 		}
 	}
@@ -144,44 +142,6 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (h *aliasHandler) createArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) bool {
-	if h.archiveMaxWorkers > 0 {
-		for {
-			current := h.archiveWorkers.Load()
-			if current >= h.archiveMaxWorkers {
-				data.Status = http.StatusTooManyRequests
-				return false
-			}
-			if h.archiveWorkers.CompareAndSwap(current, current+1) {
-				break
-			}
-		}
-
-		defer func() {
-			for {
-				current := h.archiveWorkers.Load()
-				if current == 0 {
-					break // prevent underflow just in case
-				}
-				if h.archiveWorkers.CompareAndSwap(current, current-1) {
-					break
-				}
-			}
-		}()
-	}
-
-	switch session.archiveFormat {
-	case tarFmt:
-		return h.tar(w, r, session, data)
-	case tgzFmt:
-		return h.tgz(w, r, session, data)
-	case zipFmt:
-		return h.zip(w, r, session, data)
-	}
-
-	return false
-}
-
 func newAliasHandler(
 	p *param.Param,
 	vhostCtx *vhostContext,
@@ -209,8 +169,8 @@ func newAliasHandler(
 		toHttpsPort:  p.ToHttpsPort,
 		defaultSort:  p.DefaultSort,
 
-		archiveMaxWorkers: vhostCtx.archiveMaxWorkers,
-		archiveWorkers:    vhostCtx.archiveWorkers,
+		archiveWorkersMax: p.ArchiveWorkersMax,
+		archivingWorkers:  vhostCtx.archivingWorkers,
 
 		users:  vhostCtx.users,
 		theme:  vhostCtx.theme,

src/serverHandler/archive.go

@@ -1,13 +1,13 @@
 package serverHandler
 
 import (
+	"mjpclab.dev/ghfs/src/util"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"strings"
-
-	"mjpclab.dev/ghfs/src/util"
+	"sync/atomic"
 )
 
 type archiveCallback func(f *os.File, fInfo os.FileInfo, relPath string) error
@@ -211,3 +211,37 @@ func (h *aliasHandler) normalizeArchiveSelections(r *http.Request) ([]string, bo
 
 	return selections, true
 }
+
+func (h *aliasHandler) startArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) bool {
+	switch session.archiveFormat {
+	case tarFmt:
+		return h.tar(w, r, session, data)
+	case tgzFmt:
+		return h.tgz(w, r, session, data)
+	case zipFmt:
+		return h.zip(w, r, session, data)
+	}
+
+	return false
+}
+
+func (h *aliasHandler) tryArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) (ok bool) {
+	if h.archivingWorkers == nil {
+		return h.startArchive(w, r, session, data)
+	}
+
+	if *h.archivingWorkers >= h.archiveWorkersMax {
+		data.Status = http.StatusTooManyRequests
+		return false
+	}
+
+	archiving := atomic.AddUint32(h.archivingWorkers, 1)
+	ok = archiving <= h.archiveWorkersMax
+	if ok {
+		ok = h.startArchive(w, r, session, data)
+	} else {
+		data.Status = http.StatusTooManyRequests
+	}
+	atomic.AddUint32(h.archivingWorkers, ^uint32(0)) // archiveWorkers -= 1
+	return
+}

src/serverHandler/vhostHandler.go

@@ -1,16 +1,14 @@
 package serverHandler
 
 import (
-	"net/http"
-	"regexp"
-	"sync/atomic"
-
 	"mjpclab.dev/ghfs/src/param"
 	"mjpclab.dev/ghfs/src/serverError"
 	"mjpclab.dev/ghfs/src/serverLog"
 	"mjpclab.dev/ghfs/src/tpl/defaultTheme"
 	"mjpclab.dev/ghfs/src/tpl/theme"
 	"mjpclab.dev/ghfs/src/user"
+	"net/http"
+	"regexp"
 )
 
 type vhostContext struct {
@@ -31,8 +29,7 @@ type vhostContext struct {
 	archiveUrlsUsers pathIntsList
 	archiveDirsUsers pathIntsList
 
-	archiveMaxWorkers uint32
-	archiveWorkers    *atomic.Uint32
+	archivingWorkers *uint32
 
 	shows     *regexp.Regexp
 	showDirs  *regexp.Regexp
@@ -100,6 +97,12 @@ func NewVhostHandler(
 		theme = defaultTheme.DefaultTheme
 	}
 
+	var archivingWorkersValue uint32
+	var archivingWorkers *uint32
+	if p.ArchiveWorkersMax > 0 {
+		archivingWorkers = &archivingWorkersValue
+	}
+
 	// alias param
 	vhostCtx := &vhostContext{
 		logger: logger,
@@ -119,8 +122,7 @@ func NewVhostHandler(
 		archiveUrlsUsers: pathUsernamesToPathUids(users, p.ArchiveUrlsUsers),
 		archiveDirsUsers: pathUsernamesToPathUids(users, p.ArchiveDirsUsers),
 
-		archiveMaxWorkers: p.ArchiveMaxWorkers,
-		archiveWorkers:    &atomic.Uint32{},
+		archivingWorkers: archivingWorkers,
 
 		shows:     shows,
 		showDirs:  showDirs,