add crawl type indicator to admin status page

This commit is contained in:
Cyberes 2023-12-12 17:52:03 -07:00
parent a39b3ea010
commit 8d08f04a4f
2 changed files with 40 additions and 15 deletions

View File

@ -22,12 +22,14 @@ type ActiveCrawl struct {
Path string `json:"path"` Path string `json:"path"`
Start int64 `json:"start"` Start int64 `json:"start"`
Elapsed int64 `json:"elapsed"` Elapsed int64 `json:"elapsed"`
Function string `json:"function"`
} }
type FinishedCrawl struct { type FinishedCrawl struct {
Path string `json:"path"` Path string `json:"path"`
Start int64 `json:"start"` Start int64 `json:"start"`
Elapsed int64 `json:"elapsed"` Elapsed int64 `json:"elapsed"`
Function string `json:"function"`
} }
type DirectoryCrawler struct { type DirectoryCrawler struct {
@ -87,11 +89,16 @@ func isSubpath(path, subpath string) bool {
return true return true
} }
func (dc *DirectoryCrawler) startCrawl(path string) bool { func (dc *DirectoryCrawler) startCrawl(path string, function string) bool {
if dc.IsCrawlActive(path) { if dc.IsCrawlActive(path) {
return false return false
} }
activeCrawls[path] = &ActiveCrawl{Path: path, Start: time.Now().Unix(), Elapsed: int64(0)} activeCrawls[path] = &ActiveCrawl{
Path: path,
Start: time.Now().Unix(),
Elapsed: int64(0),
Function: function,
}
return true return true
} }
@ -103,7 +110,12 @@ func (dc *DirectoryCrawler) endCrawl(path string) {
if len(finishedCrawls) >= maxFinishedCrawls { if len(finishedCrawls) >= maxFinishedCrawls {
finishedCrawls = finishedCrawls[1:] finishedCrawls = finishedCrawls[1:]
} }
finishedCrawls = append(finishedCrawls, FinishedCrawl{Path: path, Start: activeCrawls[path].Start, Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds())}) finishedCrawls = append(finishedCrawls, FinishedCrawl{
Path: path,
Start: activeCrawls[path].Start,
Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds()),
Function: activeCrawls[path].Function,
})
delete(activeCrawls, path) delete(activeCrawls, path)
} }

View File

@ -10,6 +10,9 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"runtime"
"strings"
) )
func (dc *DirectoryCrawler) walkRecursiveFunc(fullPath string, info os.FileInfo, err error) error { func (dc *DirectoryCrawler) walkRecursiveFunc(fullPath string, info os.FileInfo, err error) error {
@ -39,16 +42,26 @@ func (dc *DirectoryCrawler) walkNonRecursiveFunc(fullPath string, dir os.DirEntr
func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.FileInfo, error) error) error { func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.FileInfo, error) error) error {
CacheItem.RetardCheck(fullPath) CacheItem.RetardCheck(fullPath)
readyToStart := dc.startCrawl(fullPath)
// Set default value.
if walkFunc == nil {
walkFunc = dc.walkRecursiveFunc
}
//Extrapolate the name of the callback function.
pc := reflect.ValueOf(walkFunc).Pointer()
fn := runtime.FuncForPC(pc)
fullName := fn.Name()
parts := strings.Split(fullName, ".")
funcName := parts[len(parts)-1]
cleanFuncName := strings.TrimSuffix(funcName, "-fm")
readyToStart := dc.startCrawl(fullPath, strings.TrimSuffix(cleanFuncName, "Func"))
if !readyToStart { if !readyToStart {
return errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath)) return errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
} }
defer dc.endCrawl(fullPath) defer dc.endCrawl(fullPath)
if walkFunc == nil {
walkFunc = dc.walkRecursiveFunc
}
info, err := os.Lstat(fullPath) info, err := os.Lstat(fullPath)
if os.IsNotExist(err) { if os.IsNotExist(err) {
// If the path doesn't exist, just silently exit // If the path doesn't exist, just silently exit
@ -93,7 +106,7 @@ func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.File
// CrawlNoRecursion this function crawls a file or directory and does not recurse into any subdirectories. Also returns the result of the crawl. // CrawlNoRecursion this function crawls a file or directory and does not recurse into any subdirectories. Also returns the result of the crawl.
func (dc *DirectoryCrawler) CrawlNoRecursion(fullPath string) (*CacheItem.Item, error) { func (dc *DirectoryCrawler) CrawlNoRecursion(fullPath string) (*CacheItem.Item, error) {
CacheItem.RetardCheck(fullPath) CacheItem.RetardCheck(fullPath)
readyToStart := dc.startCrawl(fullPath) readyToStart := dc.startCrawl(fullPath, "walkNonRecursive")
if !readyToStart { if !readyToStart {
return nil, errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath)) return nil, errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
} }