From 6c348d99d5c5759da3a0f5d909f36bb5e56f4792 Mon Sep 17 00:00:00 2001 From: Graham Steffaniak Date: Sat, 12 Aug 2023 14:41:59 -0500 Subject: [PATCH] updated --- backend/run_benchmark.sh | 2 +- backend/run_tests.sh | 3 -- backend/search/conditions.go | 19 +++++++----- backend/search/indexing.go | 58 ++++++++++++----------------------- backend/search/search_test.go | 29 +++++++++++++++--- 5 files changed, 57 insertions(+), 54 deletions(-) diff --git a/backend/run_benchmark.sh b/backend/run_benchmark.sh index fda0f117..00d8aac8 100755 --- a/backend/run_benchmark.sh +++ b/backend/run_benchmark.sh @@ -9,7 +9,7 @@ checkExit() { if command -v go &> /dev/null then - printf "\n == Running benchmark (sends to results.txt) == \n" + printf "\n == Running benchmark == \n" go test -bench=. -benchmem ./... checkExit else diff --git a/backend/run_tests.sh b/backend/run_tests.sh index f7a6732a..f20b80b0 100755 --- a/backend/run_tests.sh +++ b/backend/run_tests.sh @@ -12,9 +12,6 @@ then printf "\n == Running tests == \n" go test -race -v ./... checkExit - printf "\n == Running benchmark (sends to results.txt) == \n" - go test -bench=. -benchtime=100x -benchmem ./... - checkExit else echo "ERROR: unable to perform tests" exit 1 diff --git a/backend/search/conditions.go b/backend/search/conditions.go index 57f5a511..b6791433 100644 --- a/backend/search/conditions.go +++ b/backend/search/conditions.go @@ -26,9 +26,10 @@ var compressedFile = []string{ } type SearchOptions struct { - Conditions map[string]bool - Size int - Terms []string + Conditions map[string]bool + LargerThan int + SmallerThan int + Terms []string } func ParseSearch(value string) *SearchOptions { @@ -68,13 +69,15 @@ func ParseSearch(value string) *SearchOptions { if len(filter) < 8 { continue } - if filter[:7] == "larger=" { + if strings.HasPrefix(filter, "largerThan=") { opts.Conditions["larger"] = true - opts.Size = updateSize(filter[7:]) // everything after larger= + size := strings.TrimPrefix(filter, "largerThan=") + opts.LargerThan = updateSize(size) } - if filter[:8] == "smaller=" { - opts.Conditions["smaller"] = true - opts.Size = updateSize(filter[8:]) // everything after smaller= + if strings.HasPrefix(filter, "smallerThan=") { + opts.Conditions["larger"] = true + size := strings.TrimPrefix(filter, "smallerThan=") + opts.SmallerThan = updateSize(size) } } diff --git a/backend/search/indexing.go b/backend/search/indexing.go index b58da03d..f13f7de0 100644 --- a/backend/search/indexing.go +++ b/backend/search/indexing.go @@ -95,10 +95,8 @@ func addToIndex(path string, fileName string, isDir bool) { mutex.Lock() defer mutex.Unlock() path = strings.TrimPrefix(path, rootPath+"/") - path = strings.TrimSuffix(path, "/") + "/" - if path == "" { - path = "/" - } + path = strings.TrimSuffix(path, "/") + path = "/" + strings.TrimPrefix(path, "/") if isDir { indexes[path] = []string{} } else { @@ -110,20 +108,19 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str sourceSession := "0.0.0.0" runningHash := generateRandomHash(4) sessionInProgress.Store(sourceSession, runningHash) // Store the value in the sync.Map - searchOptions := ParseSearch(search) mutex.RLock() defer mutex.RUnlock() fileListTypes := make(map[string]map[string]bool) var matching []string + var matches bool + var fileType map[string]bool // 250 items total seems like a reasonable limit maximum := 250 for _, searchTerm := range searchOptions.Terms { if searchTerm == "" { continue } - // Create a reused fileType map - reusedFileType := map[string]bool{} // Iterate over the indexes count := 0 for pathName, files := range indexes { @@ -136,12 +133,15 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str if !found || value != runningHash { return []string{}, map[string]map[string]bool{} } - pathName = scopedPathNameFilter(pathName, scope) - if pathName == "" { + if pathName != "/" { + pathName = pathName + "/" + } + if !strings.HasPrefix(pathName, scope) { + // skip directory if not in scope continue } // check if dir matches - matches, fileType := containsSearchTerm(pathName, searchTerm, *searchOptions, false) + matches, fileType = containsSearchTerm(pathName, searchTerm, *searchOptions, true) if matches { matching = append(matching, pathName) fileListTypes[pathName] = fileType @@ -154,14 +154,7 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str continue } matching = append(matching, pathName+fileName) - // Clear and reuse the fileType map - for key := range reusedFileType { - delete(reusedFileType, key) - } - for key, value := range fileType { - reusedFileType[key] = value - } - fileListTypes[pathName] = copyFileTypeMap(reusedFileType) + fileListTypes[pathName+fileName] = fileType count++ } } @@ -175,17 +168,10 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str return matching, fileListTypes } -func copyFileTypeMap(src map[string]bool) map[string]bool { - dest := make(map[string]bool, len(src)) - for key, value := range src { - dest[key] = value - } - return dest -} - -func scopedPathNameFilter(pathName string, scope string) string { +func inSearchScope(pathName string, scope string) string { if strings.HasPrefix(pathName, scope) { - return strings.TrimPrefix(pathName, scope) + pathName = strings.TrimPrefix(pathName, scope) + return pathName } return "" } @@ -194,13 +180,12 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption conditions := options.Conditions path := getLastPathComponent(pathName) // Convert to lowercase once - lowerPath := path lowerSearchTerm := searchTerm if !conditions["exact"] { - lowerPath = strings.ToLower(path) + path = strings.ToLower(path) lowerSearchTerm = strings.ToLower(searchTerm) } - if strings.Contains(lowerPath, lowerSearchTerm) { + if strings.Contains(path, lowerSearchTerm) { // Reuse the fileTypes map and clear its values fileTypes := map[string]bool{ "audio": false, @@ -210,15 +195,14 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption "archive": false, "dir": false, } - // Calculate fileSize only if needed var fileSize int64 if conditions["larger"] || conditions["smaller"] { + log.Println(conditions) fileSize = getFileSize(pathName) } - matchesAllConditions := true - extension := filepath.Ext(lowerPath) + extension := filepath.Ext(path) mimetype := mime.TypeByExtension(extension) fileTypes["audio"] = strings.HasPrefix(mimetype, "audio") fileTypes["image"] = strings.HasPrefix(mimetype, "image") @@ -226,7 +210,6 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption fileTypes["doc"] = isDoc(extension) fileTypes["archive"] = isArchive(extension) fileTypes["dir"] = isDir - for t, v := range conditions { if t == "exact" { continue @@ -234,9 +217,9 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption var matchesCondition bool switch t { case "larger": - matchesCondition = fileSize > int64(options.Size)*1000000 + matchesCondition = fileSize > int64(options.LargerThan)*1000000 case "smaller": - matchesCondition = fileSize < int64(options.Size)*1000000 + matchesCondition = fileSize < int64(options.SmallerThan)*1000000 default: matchesCondition = v == fileTypes[t] } @@ -244,7 +227,6 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption matchesAllConditions = false } } - return matchesAllConditions, fileTypes } // Clear variables and return diff --git a/backend/search/search_test.go b/backend/search/search_test.go index 6399d2fe..2db409a4 100644 --- a/backend/search/search_test.go +++ b/backend/search/search_test.go @@ -31,6 +31,30 @@ func TestParseSearch(t *testing.T) { if !reflect.DeepEqual(value, want) { t.Fatalf("\n got: %+v\n want: %+v", value, want) } + value = ParseSearch("type:largerThan=100 type:smallerThan=1000 test") + want = &SearchOptions{ + Conditions: map[string]bool{ + "exact": false, + "larger": true, + }, + Terms: []string{"test"}, + LargerThan: 100, + SmallerThan: 1000, + } + if !reflect.DeepEqual(value, want) { + t.Fatalf("\n got: %+v\n want: %+v", value, want) + } + value = ParseSearch("type:audio thisfile") + want = &SearchOptions{ + Conditions: map[string]bool{ + "exact": false, + "audio": true, + }, + Terms: []string{"thisfile"}, + } + if !reflect.DeepEqual(value, want) { + t.Fatalf("\n got: %+v\n want: %+v", value, want) + } } func BenchmarkSearchAllIndexes(b *testing.B) { @@ -60,15 +84,12 @@ func BenchmarkFillIndex(b *testing.B) { for i := 0; i < b.N; i++ { createMockData(10000, 10) // 1000 dirs, 3 files per dir } - for a, _ := range indexes { - b.Logf(a) - } printBenchmarkResults(b) } func createMockData(numDirs, numFilesPerDir int) { for i := 0; i < numDirs; i++ { - dirName := getRandomTerm() + dirName := "srv/" + getRandomTerm() addToIndex("/", dirName, true) for j := 0; j < numFilesPerDir; j++ { fileName := "file-" + getRandomTerm() + getRandomExtension()