add crawl type indicator to admin status page

This commit is contained in:
Cyberes 2023-12-12 17:52:03 -07:00
parent a39b3ea010
commit 8d08f04a4f
2 changed files with 40 additions and 15 deletions

View File

@ -19,15 +19,17 @@ var activeCrawlsMutex = &sync.Mutex{}
var finishedCrawlsMutex = &sync.Mutex{}
type ActiveCrawl struct {
Path string `json:"path"`
Start int64 `json:"start"`
Elapsed int64 `json:"elapsed"`
Path string `json:"path"`
Start int64 `json:"start"`
Elapsed int64 `json:"elapsed"`
Function string `json:"function"`
}
type FinishedCrawl struct {
Path string `json:"path"`
Start int64 `json:"start"`
Elapsed int64 `json:"elapsed"`
Path string `json:"path"`
Start int64 `json:"start"`
Elapsed int64 `json:"elapsed"`
Function string `json:"function"`
}
type DirectoryCrawler struct {
@ -87,11 +89,16 @@ func isSubpath(path, subpath string) bool {
return true
}
func (dc *DirectoryCrawler) startCrawl(path string) bool {
func (dc *DirectoryCrawler) startCrawl(path string, function string) bool {
if dc.IsCrawlActive(path) {
return false
}
activeCrawls[path] = &ActiveCrawl{Path: path, Start: time.Now().Unix(), Elapsed: int64(0)}
activeCrawls[path] = &ActiveCrawl{
Path: path,
Start: time.Now().Unix(),
Elapsed: int64(0),
Function: function,
}
return true
}
@ -103,7 +110,12 @@ func (dc *DirectoryCrawler) endCrawl(path string) {
if len(finishedCrawls) >= maxFinishedCrawls {
finishedCrawls = finishedCrawls[1:]
}
finishedCrawls = append(finishedCrawls, FinishedCrawl{Path: path, Start: activeCrawls[path].Start, Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds())})
finishedCrawls = append(finishedCrawls, FinishedCrawl{
Path: path,
Start: activeCrawls[path].Start,
Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds()),
Function: activeCrawls[path].Function,
})
delete(activeCrawls, path)
}

View File

@ -10,6 +10,9 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
)
func (dc *DirectoryCrawler) walkRecursiveFunc(fullPath string, info os.FileInfo, err error) error {
@ -39,16 +42,26 @@ func (dc *DirectoryCrawler) walkNonRecursiveFunc(fullPath string, dir os.DirEntr
func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.FileInfo, error) error) error {
CacheItem.RetardCheck(fullPath)
readyToStart := dc.startCrawl(fullPath)
// Set default value.
if walkFunc == nil {
walkFunc = dc.walkRecursiveFunc
}
//Extrapolate the name of the callback function.
pc := reflect.ValueOf(walkFunc).Pointer()
fn := runtime.FuncForPC(pc)
fullName := fn.Name()
parts := strings.Split(fullName, ".")
funcName := parts[len(parts)-1]
cleanFuncName := strings.TrimSuffix(funcName, "-fm")
readyToStart := dc.startCrawl(fullPath, strings.TrimSuffix(cleanFuncName, "Func"))
if !readyToStart {
return errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
}
defer dc.endCrawl(fullPath)
if walkFunc == nil {
walkFunc = dc.walkRecursiveFunc
}
info, err := os.Lstat(fullPath)
if os.IsNotExist(err) {
// If the path doesn't exist, just silently exit
@ -93,7 +106,7 @@ func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.File
// CrawlNoRecursion this function crawls a file or directory and does not recurse into any subdirectories. Also returns the result of the crawl.
func (dc *DirectoryCrawler) CrawlNoRecursion(fullPath string) (*CacheItem.Item, error) {
CacheItem.RetardCheck(fullPath)
readyToStart := dc.startCrawl(fullPath)
readyToStart := dc.startCrawl(fullPath, "walkNonRecursive")
if !readyToStart {
return nil, errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
}