updated
This commit is contained in:
parent
4e01929dc8
commit
6c348d99d5
|
@ -9,7 +9,7 @@ checkExit() {
|
||||||
|
|
||||||
if command -v go &> /dev/null
|
if command -v go &> /dev/null
|
||||||
then
|
then
|
||||||
printf "\n == Running benchmark (sends to results.txt) == \n"
|
printf "\n == Running benchmark == \n"
|
||||||
go test -bench=. -benchmem ./...
|
go test -bench=. -benchmem ./...
|
||||||
checkExit
|
checkExit
|
||||||
else
|
else
|
||||||
|
|
|
@ -12,9 +12,6 @@ then
|
||||||
printf "\n == Running tests == \n"
|
printf "\n == Running tests == \n"
|
||||||
go test -race -v ./...
|
go test -race -v ./...
|
||||||
checkExit
|
checkExit
|
||||||
printf "\n == Running benchmark (sends to results.txt) == \n"
|
|
||||||
go test -bench=. -benchtime=100x -benchmem ./...
|
|
||||||
checkExit
|
|
||||||
else
|
else
|
||||||
echo "ERROR: unable to perform tests"
|
echo "ERROR: unable to perform tests"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -26,9 +26,10 @@ var compressedFile = []string{
|
||||||
}
|
}
|
||||||
|
|
||||||
type SearchOptions struct {
|
type SearchOptions struct {
|
||||||
Conditions map[string]bool
|
Conditions map[string]bool
|
||||||
Size int
|
LargerThan int
|
||||||
Terms []string
|
SmallerThan int
|
||||||
|
Terms []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseSearch(value string) *SearchOptions {
|
func ParseSearch(value string) *SearchOptions {
|
||||||
|
@ -68,13 +69,15 @@ func ParseSearch(value string) *SearchOptions {
|
||||||
if len(filter) < 8 {
|
if len(filter) < 8 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if filter[:7] == "larger=" {
|
if strings.HasPrefix(filter, "largerThan=") {
|
||||||
opts.Conditions["larger"] = true
|
opts.Conditions["larger"] = true
|
||||||
opts.Size = updateSize(filter[7:]) // everything after larger=
|
size := strings.TrimPrefix(filter, "largerThan=")
|
||||||
|
opts.LargerThan = updateSize(size)
|
||||||
}
|
}
|
||||||
if filter[:8] == "smaller=" {
|
if strings.HasPrefix(filter, "smallerThan=") {
|
||||||
opts.Conditions["smaller"] = true
|
opts.Conditions["larger"] = true
|
||||||
opts.Size = updateSize(filter[8:]) // everything after smaller=
|
size := strings.TrimPrefix(filter, "smallerThan=")
|
||||||
|
opts.SmallerThan = updateSize(size)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -95,10 +95,8 @@ func addToIndex(path string, fileName string, isDir bool) {
|
||||||
mutex.Lock()
|
mutex.Lock()
|
||||||
defer mutex.Unlock()
|
defer mutex.Unlock()
|
||||||
path = strings.TrimPrefix(path, rootPath+"/")
|
path = strings.TrimPrefix(path, rootPath+"/")
|
||||||
path = strings.TrimSuffix(path, "/") + "/"
|
path = strings.TrimSuffix(path, "/")
|
||||||
if path == "" {
|
path = "/" + strings.TrimPrefix(path, "/")
|
||||||
path = "/"
|
|
||||||
}
|
|
||||||
if isDir {
|
if isDir {
|
||||||
indexes[path] = []string{}
|
indexes[path] = []string{}
|
||||||
} else {
|
} else {
|
||||||
|
@ -110,20 +108,19 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
|
||||||
sourceSession := "0.0.0.0"
|
sourceSession := "0.0.0.0"
|
||||||
runningHash := generateRandomHash(4)
|
runningHash := generateRandomHash(4)
|
||||||
sessionInProgress.Store(sourceSession, runningHash) // Store the value in the sync.Map
|
sessionInProgress.Store(sourceSession, runningHash) // Store the value in the sync.Map
|
||||||
|
|
||||||
searchOptions := ParseSearch(search)
|
searchOptions := ParseSearch(search)
|
||||||
mutex.RLock()
|
mutex.RLock()
|
||||||
defer mutex.RUnlock()
|
defer mutex.RUnlock()
|
||||||
fileListTypes := make(map[string]map[string]bool)
|
fileListTypes := make(map[string]map[string]bool)
|
||||||
var matching []string
|
var matching []string
|
||||||
|
var matches bool
|
||||||
|
var fileType map[string]bool
|
||||||
// 250 items total seems like a reasonable limit
|
// 250 items total seems like a reasonable limit
|
||||||
maximum := 250
|
maximum := 250
|
||||||
for _, searchTerm := range searchOptions.Terms {
|
for _, searchTerm := range searchOptions.Terms {
|
||||||
if searchTerm == "" {
|
if searchTerm == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Create a reused fileType map
|
|
||||||
reusedFileType := map[string]bool{}
|
|
||||||
// Iterate over the indexes
|
// Iterate over the indexes
|
||||||
count := 0
|
count := 0
|
||||||
for pathName, files := range indexes {
|
for pathName, files := range indexes {
|
||||||
|
@ -136,12 +133,15 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
|
||||||
if !found || value != runningHash {
|
if !found || value != runningHash {
|
||||||
return []string{}, map[string]map[string]bool{}
|
return []string{}, map[string]map[string]bool{}
|
||||||
}
|
}
|
||||||
pathName = scopedPathNameFilter(pathName, scope)
|
if pathName != "/" {
|
||||||
if pathName == "" {
|
pathName = pathName + "/"
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(pathName, scope) {
|
||||||
|
// skip directory if not in scope
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// check if dir matches
|
// check if dir matches
|
||||||
matches, fileType := containsSearchTerm(pathName, searchTerm, *searchOptions, false)
|
matches, fileType = containsSearchTerm(pathName, searchTerm, *searchOptions, true)
|
||||||
if matches {
|
if matches {
|
||||||
matching = append(matching, pathName)
|
matching = append(matching, pathName)
|
||||||
fileListTypes[pathName] = fileType
|
fileListTypes[pathName] = fileType
|
||||||
|
@ -154,14 +154,7 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
matching = append(matching, pathName+fileName)
|
matching = append(matching, pathName+fileName)
|
||||||
// Clear and reuse the fileType map
|
fileListTypes[pathName+fileName] = fileType
|
||||||
for key := range reusedFileType {
|
|
||||||
delete(reusedFileType, key)
|
|
||||||
}
|
|
||||||
for key, value := range fileType {
|
|
||||||
reusedFileType[key] = value
|
|
||||||
}
|
|
||||||
fileListTypes[pathName] = copyFileTypeMap(reusedFileType)
|
|
||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -175,17 +168,10 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
|
||||||
return matching, fileListTypes
|
return matching, fileListTypes
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyFileTypeMap(src map[string]bool) map[string]bool {
|
func inSearchScope(pathName string, scope string) string {
|
||||||
dest := make(map[string]bool, len(src))
|
|
||||||
for key, value := range src {
|
|
||||||
dest[key] = value
|
|
||||||
}
|
|
||||||
return dest
|
|
||||||
}
|
|
||||||
|
|
||||||
func scopedPathNameFilter(pathName string, scope string) string {
|
|
||||||
if strings.HasPrefix(pathName, scope) {
|
if strings.HasPrefix(pathName, scope) {
|
||||||
return strings.TrimPrefix(pathName, scope)
|
pathName = strings.TrimPrefix(pathName, scope)
|
||||||
|
return pathName
|
||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
@ -194,13 +180,12 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
|
||||||
conditions := options.Conditions
|
conditions := options.Conditions
|
||||||
path := getLastPathComponent(pathName)
|
path := getLastPathComponent(pathName)
|
||||||
// Convert to lowercase once
|
// Convert to lowercase once
|
||||||
lowerPath := path
|
|
||||||
lowerSearchTerm := searchTerm
|
lowerSearchTerm := searchTerm
|
||||||
if !conditions["exact"] {
|
if !conditions["exact"] {
|
||||||
lowerPath = strings.ToLower(path)
|
path = strings.ToLower(path)
|
||||||
lowerSearchTerm = strings.ToLower(searchTerm)
|
lowerSearchTerm = strings.ToLower(searchTerm)
|
||||||
}
|
}
|
||||||
if strings.Contains(lowerPath, lowerSearchTerm) {
|
if strings.Contains(path, lowerSearchTerm) {
|
||||||
// Reuse the fileTypes map and clear its values
|
// Reuse the fileTypes map and clear its values
|
||||||
fileTypes := map[string]bool{
|
fileTypes := map[string]bool{
|
||||||
"audio": false,
|
"audio": false,
|
||||||
|
@ -210,15 +195,14 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
|
||||||
"archive": false,
|
"archive": false,
|
||||||
"dir": false,
|
"dir": false,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate fileSize only if needed
|
// Calculate fileSize only if needed
|
||||||
var fileSize int64
|
var fileSize int64
|
||||||
if conditions["larger"] || conditions["smaller"] {
|
if conditions["larger"] || conditions["smaller"] {
|
||||||
|
log.Println(conditions)
|
||||||
fileSize = getFileSize(pathName)
|
fileSize = getFileSize(pathName)
|
||||||
}
|
}
|
||||||
|
|
||||||
matchesAllConditions := true
|
matchesAllConditions := true
|
||||||
extension := filepath.Ext(lowerPath)
|
extension := filepath.Ext(path)
|
||||||
mimetype := mime.TypeByExtension(extension)
|
mimetype := mime.TypeByExtension(extension)
|
||||||
fileTypes["audio"] = strings.HasPrefix(mimetype, "audio")
|
fileTypes["audio"] = strings.HasPrefix(mimetype, "audio")
|
||||||
fileTypes["image"] = strings.HasPrefix(mimetype, "image")
|
fileTypes["image"] = strings.HasPrefix(mimetype, "image")
|
||||||
|
@ -226,7 +210,6 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
|
||||||
fileTypes["doc"] = isDoc(extension)
|
fileTypes["doc"] = isDoc(extension)
|
||||||
fileTypes["archive"] = isArchive(extension)
|
fileTypes["archive"] = isArchive(extension)
|
||||||
fileTypes["dir"] = isDir
|
fileTypes["dir"] = isDir
|
||||||
|
|
||||||
for t, v := range conditions {
|
for t, v := range conditions {
|
||||||
if t == "exact" {
|
if t == "exact" {
|
||||||
continue
|
continue
|
||||||
|
@ -234,9 +217,9 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
|
||||||
var matchesCondition bool
|
var matchesCondition bool
|
||||||
switch t {
|
switch t {
|
||||||
case "larger":
|
case "larger":
|
||||||
matchesCondition = fileSize > int64(options.Size)*1000000
|
matchesCondition = fileSize > int64(options.LargerThan)*1000000
|
||||||
case "smaller":
|
case "smaller":
|
||||||
matchesCondition = fileSize < int64(options.Size)*1000000
|
matchesCondition = fileSize < int64(options.SmallerThan)*1000000
|
||||||
default:
|
default:
|
||||||
matchesCondition = v == fileTypes[t]
|
matchesCondition = v == fileTypes[t]
|
||||||
}
|
}
|
||||||
|
@ -244,7 +227,6 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
|
||||||
matchesAllConditions = false
|
matchesAllConditions = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return matchesAllConditions, fileTypes
|
return matchesAllConditions, fileTypes
|
||||||
}
|
}
|
||||||
// Clear variables and return
|
// Clear variables and return
|
||||||
|
|
|
@ -31,6 +31,30 @@ func TestParseSearch(t *testing.T) {
|
||||||
if !reflect.DeepEqual(value, want) {
|
if !reflect.DeepEqual(value, want) {
|
||||||
t.Fatalf("\n got: %+v\n want: %+v", value, want)
|
t.Fatalf("\n got: %+v\n want: %+v", value, want)
|
||||||
}
|
}
|
||||||
|
value = ParseSearch("type:largerThan=100 type:smallerThan=1000 test")
|
||||||
|
want = &SearchOptions{
|
||||||
|
Conditions: map[string]bool{
|
||||||
|
"exact": false,
|
||||||
|
"larger": true,
|
||||||
|
},
|
||||||
|
Terms: []string{"test"},
|
||||||
|
LargerThan: 100,
|
||||||
|
SmallerThan: 1000,
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(value, want) {
|
||||||
|
t.Fatalf("\n got: %+v\n want: %+v", value, want)
|
||||||
|
}
|
||||||
|
value = ParseSearch("type:audio thisfile")
|
||||||
|
want = &SearchOptions{
|
||||||
|
Conditions: map[string]bool{
|
||||||
|
"exact": false,
|
||||||
|
"audio": true,
|
||||||
|
},
|
||||||
|
Terms: []string{"thisfile"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(value, want) {
|
||||||
|
t.Fatalf("\n got: %+v\n want: %+v", value, want)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkSearchAllIndexes(b *testing.B) {
|
func BenchmarkSearchAllIndexes(b *testing.B) {
|
||||||
|
@ -60,15 +84,12 @@ func BenchmarkFillIndex(b *testing.B) {
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
createMockData(10000, 10) // 1000 dirs, 3 files per dir
|
createMockData(10000, 10) // 1000 dirs, 3 files per dir
|
||||||
}
|
}
|
||||||
for a, _ := range indexes {
|
|
||||||
b.Logf(a)
|
|
||||||
}
|
|
||||||
printBenchmarkResults(b)
|
printBenchmarkResults(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createMockData(numDirs, numFilesPerDir int) {
|
func createMockData(numDirs, numFilesPerDir int) {
|
||||||
for i := 0; i < numDirs; i++ {
|
for i := 0; i < numDirs; i++ {
|
||||||
dirName := getRandomTerm()
|
dirName := "srv/" + getRandomTerm()
|
||||||
addToIndex("/", dirName, true)
|
addToIndex("/", dirName, true)
|
||||||
for j := 0; j < numFilesPerDir; j++ {
|
for j := 0; j < numFilesPerDir; j++ {
|
||||||
fileName := "file-" + getRandomTerm() + getRandomExtension()
|
fileName := "file-" + getRandomTerm() + getRandomExtension()
|
||||||
|
|
Loading…
Reference in New Issue