add crawl type indicator to admin status page
This commit is contained in:
parent
a39b3ea010
commit
8d08f04a4f
|
@ -22,12 +22,14 @@ type ActiveCrawl struct {
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Start int64 `json:"start"`
|
Start int64 `json:"start"`
|
||||||
Elapsed int64 `json:"elapsed"`
|
Elapsed int64 `json:"elapsed"`
|
||||||
|
Function string `json:"function"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type FinishedCrawl struct {
|
type FinishedCrawl struct {
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Start int64 `json:"start"`
|
Start int64 `json:"start"`
|
||||||
Elapsed int64 `json:"elapsed"`
|
Elapsed int64 `json:"elapsed"`
|
||||||
|
Function string `json:"function"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type DirectoryCrawler struct {
|
type DirectoryCrawler struct {
|
||||||
|
@ -87,11 +89,16 @@ func isSubpath(path, subpath string) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dc *DirectoryCrawler) startCrawl(path string) bool {
|
func (dc *DirectoryCrawler) startCrawl(path string, function string) bool {
|
||||||
if dc.IsCrawlActive(path) {
|
if dc.IsCrawlActive(path) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
activeCrawls[path] = &ActiveCrawl{Path: path, Start: time.Now().Unix(), Elapsed: int64(0)}
|
activeCrawls[path] = &ActiveCrawl{
|
||||||
|
Path: path,
|
||||||
|
Start: time.Now().Unix(),
|
||||||
|
Elapsed: int64(0),
|
||||||
|
Function: function,
|
||||||
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,7 +110,12 @@ func (dc *DirectoryCrawler) endCrawl(path string) {
|
||||||
if len(finishedCrawls) >= maxFinishedCrawls {
|
if len(finishedCrawls) >= maxFinishedCrawls {
|
||||||
finishedCrawls = finishedCrawls[1:]
|
finishedCrawls = finishedCrawls[1:]
|
||||||
}
|
}
|
||||||
finishedCrawls = append(finishedCrawls, FinishedCrawl{Path: path, Start: activeCrawls[path].Start, Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds())})
|
finishedCrawls = append(finishedCrawls, FinishedCrawl{
|
||||||
|
Path: path,
|
||||||
|
Start: activeCrawls[path].Start,
|
||||||
|
Elapsed: int64(time.Since(time.Unix(activeCrawls[path].Start, 0)).Seconds()),
|
||||||
|
Function: activeCrawls[path].Function,
|
||||||
|
})
|
||||||
delete(activeCrawls, path)
|
delete(activeCrawls, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,6 +10,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (dc *DirectoryCrawler) walkRecursiveFunc(fullPath string, info os.FileInfo, err error) error {
|
func (dc *DirectoryCrawler) walkRecursiveFunc(fullPath string, info os.FileInfo, err error) error {
|
||||||
|
@ -39,16 +42,26 @@ func (dc *DirectoryCrawler) walkNonRecursiveFunc(fullPath string, dir os.DirEntr
|
||||||
|
|
||||||
func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.FileInfo, error) error) error {
|
func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.FileInfo, error) error) error {
|
||||||
CacheItem.RetardCheck(fullPath)
|
CacheItem.RetardCheck(fullPath)
|
||||||
readyToStart := dc.startCrawl(fullPath)
|
|
||||||
|
// Set default value.
|
||||||
|
if walkFunc == nil {
|
||||||
|
walkFunc = dc.walkRecursiveFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
//Extrapolate the name of the callback function.
|
||||||
|
pc := reflect.ValueOf(walkFunc).Pointer()
|
||||||
|
fn := runtime.FuncForPC(pc)
|
||||||
|
fullName := fn.Name()
|
||||||
|
parts := strings.Split(fullName, ".")
|
||||||
|
funcName := parts[len(parts)-1]
|
||||||
|
cleanFuncName := strings.TrimSuffix(funcName, "-fm")
|
||||||
|
|
||||||
|
readyToStart := dc.startCrawl(fullPath, strings.TrimSuffix(cleanFuncName, "Func"))
|
||||||
if !readyToStart {
|
if !readyToStart {
|
||||||
return errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
|
return errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
|
||||||
}
|
}
|
||||||
defer dc.endCrawl(fullPath)
|
defer dc.endCrawl(fullPath)
|
||||||
|
|
||||||
if walkFunc == nil {
|
|
||||||
walkFunc = dc.walkRecursiveFunc
|
|
||||||
}
|
|
||||||
|
|
||||||
info, err := os.Lstat(fullPath)
|
info, err := os.Lstat(fullPath)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
// If the path doesn't exist, just silently exit
|
// If the path doesn't exist, just silently exit
|
||||||
|
@ -93,7 +106,7 @@ func (dc *DirectoryCrawler) Crawl(fullPath string, walkFunc func(string, os.File
|
||||||
// CrawlNoRecursion this function crawls a file or directory and does not recurse into any subdirectories. Also returns the result of the crawl.
|
// CrawlNoRecursion this function crawls a file or directory and does not recurse into any subdirectories. Also returns the result of the crawl.
|
||||||
func (dc *DirectoryCrawler) CrawlNoRecursion(fullPath string) (*CacheItem.Item, error) {
|
func (dc *DirectoryCrawler) CrawlNoRecursion(fullPath string) (*CacheItem.Item, error) {
|
||||||
CacheItem.RetardCheck(fullPath)
|
CacheItem.RetardCheck(fullPath)
|
||||||
readyToStart := dc.startCrawl(fullPath)
|
readyToStart := dc.startCrawl(fullPath, "walkNonRecursive")
|
||||||
if !readyToStart {
|
if !readyToStart {
|
||||||
return nil, errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
|
return nil, errors.New(fmt.Sprintf(`rejecting crawl, already in progress for "%s"`, fullPath))
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue