add crawl type indicator to admin status page
This commit is contained in:
parent
a39b3ea010
commit
8d08f04a4f
|
@ -19,15 +19,17 @@ var activeCrawlsMutex = &sync.Mutex{}
|
|||
var finishedCrawlsMutex = &sync.Mutex{}
|
||||
|
||||
type ActiveCrawl struct {
|
||||
Path string `json:"path"`
|
||||
Start int64 `json:"start"`
|
||||
Elapsed int64 `json:"elapsed"`
|
||||
Path string `json:"path"`
|
||||
Start int64 `json:"start"`
|
||||
Elapsed int64 `json:"elapsed"`
|
||||
Function string `json:"function"`
|
||||
}
|
||||
|
||||
type FinishedCrawl struct {
|
||||
Path string `json:"path"`
|
||||
Start int64 `json:"start"`
|
||||
Elapsed int64 `json:"elapsed"`
|
||||
Path string `json:"path"`
|
||||
Start int64 `json:"start"`
|
||||
Elapsed int64 `json:"elapsed"`
|
||||
Function string `json:"function"`
|
||||
}
|
||||
|
||||
type DirectoryCrawler struct {
|
||||
|
@ -87,11 +89,16 @@ func isSubpath(path, subpath string) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func (dc *DirectoryCrawler) startCrawl(path string) bool {
|
||||
func (dc *DirectoryCrawler) startCrawl(path string, function string) bool {
|
||||
if dc.IsCrawlActive(path) {
|
||||
return false
|
||||
}
|
||||
activeCrawls[path] = &ActiveCrawl{Path: path, Start: time.Now().Unix(), Elapsed: int64(0)}
|
||||
activeCrawls[path] = &ActiveCrawl{
|
||||
Path: path,
|
||||
Start: time.Now().Unix(),
|
||||
Elapsed: int64(0),
|
||||
Function: function,
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
@ -103,7 +110,12 @@ func (dc *DirectoryCrawler) endCrawl(path string) {
|
|||
if len(finishedCrawls) >= maxFinishedCrawls {
|
||||
finishedCrawls = finishedCrawls[1:]
|
||||
}
|
||||
finishedCrawls = append(finishedCrawls, FinishedCrawl{Path: path, Start: activeCrawls[path].Start, Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds())})
|
||||
finishedCrawls = append(finishedCrawls, FinishedCrawl{
|
||||
Path: path,
|
||||
Start: activeCrawls[path].Start,
|
||||
Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds()),
|
||||
Function: activeCrawls[path].Function,
|
||||
})
|
||||
delete(activeCrawls, path)
|
||||
}
|
||||
|
||||
|
|
|
@ -10,6 +10,9 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (dc *DirectoryCrawler) walkRecursiveFunc(fullPath string, info os.FileInfo, err error) error {
|
||||
|
@ -39,16 +42,26 @@ func (dc *DirectoryCrawler) walkNonRecursiveFunc(fullPath string, dir os.DirEntr
|
|||
|
||||
func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.FileInfo, error) error) error {
|
||||
CacheItem.RetardCheck(fullPath)
|
||||
readyToStart := dc.startCrawl(fullPath)
|
||||
|
||||
// Set default value.
|
||||
if walkFunc == nil {
|
||||
walkFunc = dc.walkRecursiveFunc
|
||||
}
|
||||
|
||||
//Extrapolate the name of the callback function.
|
||||
pc := reflect.ValueOf(walkFunc).Pointer()
|
||||
fn := runtime.FuncForPC(pc)
|
||||
fullName := fn.Name()
|
||||
parts := strings.Split(fullName, ".")
|
||||
funcName := parts[len(parts)-1]
|
||||
cleanFuncName := strings.TrimSuffix(funcName, "-fm")
|
||||
|
||||
readyToStart := dc.startCrawl(fullPath, strings.TrimSuffix(cleanFuncName, "Func"))
|
||||
if !readyToStart {
|
||||
return errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
|
||||
}
|
||||
defer dc.endCrawl(fullPath)
|
||||
|
||||
if walkFunc == nil {
|
||||
walkFunc = dc.walkRecursiveFunc
|
||||
}
|
||||
|
||||
info, err := os.Lstat(fullPath)
|
||||
if os.IsNotExist(err) {
|
||||
// If the path doesn't exist, just silently exit
|
||||
|
@ -93,7 +106,7 @@ func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.File
|
|||
// CrawlNoRecursion this function crawls a file or directory and does not recurse into any subdirectories. Also returns the result of the crawl.
|
||||
func (dc *DirectoryCrawler) CrawlNoRecursion(fullPath string) (*CacheItem.Item, error) {
|
||||
CacheItem.RetardCheck(fullPath)
|
||||
readyToStart := dc.startCrawl(fullPath)
|
||||
readyToStart := dc.startCrawl(fullPath, "walkNonRecursive")
|
||||
if !readyToStart {
|
||||
return nil, errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue