This commit is contained in:
Graham Steffaniak 2023-08-12 14:41:59 -05:00
parent 4e01929dc8
commit 6c348d99d5
5 changed files with 57 additions and 54 deletions

View File

@ -9,7 +9,7 @@ checkExit() {
if command -v go &> /dev/null if command -v go &> /dev/null
then then
printf "\n == Running benchmark (sends to results.txt) == \n" printf "\n == Running benchmark == \n"
go test -bench=. -benchmem ./... go test -bench=. -benchmem ./...
checkExit checkExit
else else

View File

@ -12,9 +12,6 @@ then
printf "\n == Running tests == \n" printf "\n == Running tests == \n"
go test -race -v ./... go test -race -v ./...
checkExit checkExit
printf "\n == Running benchmark (sends to results.txt) == \n"
go test -bench=. -benchtime=100x -benchmem ./...
checkExit
else else
echo "ERROR: unable to perform tests" echo "ERROR: unable to perform tests"
exit 1 exit 1

View File

@ -26,9 +26,10 @@ var compressedFile = []string{
} }
type SearchOptions struct { type SearchOptions struct {
Conditions map[string]bool Conditions map[string]bool
Size int LargerThan int
Terms []string SmallerThan int
Terms []string
} }
func ParseSearch(value string) *SearchOptions { func ParseSearch(value string) *SearchOptions {
@ -68,13 +69,15 @@ func ParseSearch(value string) *SearchOptions {
if len(filter) < 8 { if len(filter) < 8 {
continue continue
} }
if filter[:7] == "larger=" { if strings.HasPrefix(filter, "largerThan=") {
opts.Conditions["larger"] = true opts.Conditions["larger"] = true
opts.Size = updateSize(filter[7:]) // everything after larger= size := strings.TrimPrefix(filter, "largerThan=")
opts.LargerThan = updateSize(size)
} }
if filter[:8] == "smaller=" { if strings.HasPrefix(filter, "smallerThan=") {
opts.Conditions["smaller"] = true opts.Conditions["larger"] = true
opts.Size = updateSize(filter[8:]) // everything after smaller= size := strings.TrimPrefix(filter, "smallerThan=")
opts.SmallerThan = updateSize(size)
} }
} }

View File

@ -95,10 +95,8 @@ func addToIndex(path string, fileName string, isDir bool) {
mutex.Lock() mutex.Lock()
defer mutex.Unlock() defer mutex.Unlock()
path = strings.TrimPrefix(path, rootPath+"/") path = strings.TrimPrefix(path, rootPath+"/")
path = strings.TrimSuffix(path, "/") + "/" path = strings.TrimSuffix(path, "/")
if path == "" { path = "/" + strings.TrimPrefix(path, "/")
path = "/"
}
if isDir { if isDir {
indexes[path] = []string{} indexes[path] = []string{}
} else { } else {
@ -110,20 +108,19 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
sourceSession := "0.0.0.0" sourceSession := "0.0.0.0"
runningHash := generateRandomHash(4) runningHash := generateRandomHash(4)
sessionInProgress.Store(sourceSession, runningHash) // Store the value in the sync.Map sessionInProgress.Store(sourceSession, runningHash) // Store the value in the sync.Map
searchOptions := ParseSearch(search) searchOptions := ParseSearch(search)
mutex.RLock() mutex.RLock()
defer mutex.RUnlock() defer mutex.RUnlock()
fileListTypes := make(map[string]map[string]bool) fileListTypes := make(map[string]map[string]bool)
var matching []string var matching []string
var matches bool
var fileType map[string]bool
// 250 items total seems like a reasonable limit // 250 items total seems like a reasonable limit
maximum := 250 maximum := 250
for _, searchTerm := range searchOptions.Terms { for _, searchTerm := range searchOptions.Terms {
if searchTerm == "" { if searchTerm == "" {
continue continue
} }
// Create a reused fileType map
reusedFileType := map[string]bool{}
// Iterate over the indexes // Iterate over the indexes
count := 0 count := 0
for pathName, files := range indexes { for pathName, files := range indexes {
@ -136,12 +133,15 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
if !found || value != runningHash { if !found || value != runningHash {
return []string{}, map[string]map[string]bool{} return []string{}, map[string]map[string]bool{}
} }
pathName = scopedPathNameFilter(pathName, scope) if pathName != "/" {
if pathName == "" { pathName = pathName + "/"
}
if !strings.HasPrefix(pathName, scope) {
// skip directory if not in scope
continue continue
} }
// check if dir matches // check if dir matches
matches, fileType := containsSearchTerm(pathName, searchTerm, *searchOptions, false) matches, fileType = containsSearchTerm(pathName, searchTerm, *searchOptions, true)
if matches { if matches {
matching = append(matching, pathName) matching = append(matching, pathName)
fileListTypes[pathName] = fileType fileListTypes[pathName] = fileType
@ -154,14 +154,7 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
continue continue
} }
matching = append(matching, pathName+fileName) matching = append(matching, pathName+fileName)
// Clear and reuse the fileType map fileListTypes[pathName+fileName] = fileType
for key := range reusedFileType {
delete(reusedFileType, key)
}
for key, value := range fileType {
reusedFileType[key] = value
}
fileListTypes[pathName] = copyFileTypeMap(reusedFileType)
count++ count++
} }
} }
@ -175,17 +168,10 @@ func SearchAllIndexes(search string, scope string) ([]string, map[string]map[str
return matching, fileListTypes return matching, fileListTypes
} }
func copyFileTypeMap(src map[string]bool) map[string]bool { func inSearchScope(pathName string, scope string) string {
dest := make(map[string]bool, len(src))
for key, value := range src {
dest[key] = value
}
return dest
}
func scopedPathNameFilter(pathName string, scope string) string {
if strings.HasPrefix(pathName, scope) { if strings.HasPrefix(pathName, scope) {
return strings.TrimPrefix(pathName, scope) pathName = strings.TrimPrefix(pathName, scope)
return pathName
} }
return "" return ""
} }
@ -194,13 +180,12 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
conditions := options.Conditions conditions := options.Conditions
path := getLastPathComponent(pathName) path := getLastPathComponent(pathName)
// Convert to lowercase once // Convert to lowercase once
lowerPath := path
lowerSearchTerm := searchTerm lowerSearchTerm := searchTerm
if !conditions["exact"] { if !conditions["exact"] {
lowerPath = strings.ToLower(path) path = strings.ToLower(path)
lowerSearchTerm = strings.ToLower(searchTerm) lowerSearchTerm = strings.ToLower(searchTerm)
} }
if strings.Contains(lowerPath, lowerSearchTerm) { if strings.Contains(path, lowerSearchTerm) {
// Reuse the fileTypes map and clear its values // Reuse the fileTypes map and clear its values
fileTypes := map[string]bool{ fileTypes := map[string]bool{
"audio": false, "audio": false,
@ -210,15 +195,14 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
"archive": false, "archive": false,
"dir": false, "dir": false,
} }
// Calculate fileSize only if needed // Calculate fileSize only if needed
var fileSize int64 var fileSize int64
if conditions["larger"] || conditions["smaller"] { if conditions["larger"] || conditions["smaller"] {
log.Println(conditions)
fileSize = getFileSize(pathName) fileSize = getFileSize(pathName)
} }
matchesAllConditions := true matchesAllConditions := true
extension := filepath.Ext(lowerPath) extension := filepath.Ext(path)
mimetype := mime.TypeByExtension(extension) mimetype := mime.TypeByExtension(extension)
fileTypes["audio"] = strings.HasPrefix(mimetype, "audio") fileTypes["audio"] = strings.HasPrefix(mimetype, "audio")
fileTypes["image"] = strings.HasPrefix(mimetype, "image") fileTypes["image"] = strings.HasPrefix(mimetype, "image")
@ -226,7 +210,6 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
fileTypes["doc"] = isDoc(extension) fileTypes["doc"] = isDoc(extension)
fileTypes["archive"] = isArchive(extension) fileTypes["archive"] = isArchive(extension)
fileTypes["dir"] = isDir fileTypes["dir"] = isDir
for t, v := range conditions { for t, v := range conditions {
if t == "exact" { if t == "exact" {
continue continue
@ -234,9 +217,9 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
var matchesCondition bool var matchesCondition bool
switch t { switch t {
case "larger": case "larger":
matchesCondition = fileSize > int64(options.Size)*1000000 matchesCondition = fileSize > int64(options.LargerThan)*1000000
case "smaller": case "smaller":
matchesCondition = fileSize < int64(options.Size)*1000000 matchesCondition = fileSize < int64(options.SmallerThan)*1000000
default: default:
matchesCondition = v == fileTypes[t] matchesCondition = v == fileTypes[t]
} }
@ -244,7 +227,6 @@ func containsSearchTerm(pathName string, searchTerm string, options SearchOption
matchesAllConditions = false matchesAllConditions = false
} }
} }
return matchesAllConditions, fileTypes return matchesAllConditions, fileTypes
} }
// Clear variables and return // Clear variables and return

View File

@ -31,6 +31,30 @@ func TestParseSearch(t *testing.T) {
if !reflect.DeepEqual(value, want) { if !reflect.DeepEqual(value, want) {
t.Fatalf("\n got: %+v\n want: %+v", value, want) t.Fatalf("\n got: %+v\n want: %+v", value, want)
} }
value = ParseSearch("type:largerThan=100 type:smallerThan=1000 test")
want = &SearchOptions{
Conditions: map[string]bool{
"exact": false,
"larger": true,
},
Terms: []string{"test"},
LargerThan: 100,
SmallerThan: 1000,
}
if !reflect.DeepEqual(value, want) {
t.Fatalf("\n got: %+v\n want: %+v", value, want)
}
value = ParseSearch("type:audio thisfile")
want = &SearchOptions{
Conditions: map[string]bool{
"exact": false,
"audio": true,
},
Terms: []string{"thisfile"},
}
if !reflect.DeepEqual(value, want) {
t.Fatalf("\n got: %+v\n want: %+v", value, want)
}
} }
func BenchmarkSearchAllIndexes(b *testing.B) { func BenchmarkSearchAllIndexes(b *testing.B) {
@ -60,15 +84,12 @@ func BenchmarkFillIndex(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
createMockData(10000, 10) // 1000 dirs, 3 files per dir createMockData(10000, 10) // 1000 dirs, 3 files per dir
} }
for a, _ := range indexes {
b.Logf(a)
}
printBenchmarkResults(b) printBenchmarkResults(b)
} }
func createMockData(numDirs, numFilesPerDir int) { func createMockData(numDirs, numFilesPerDir int) {
for i := 0; i < numDirs; i++ { for i := 0; i < numDirs; i++ {
dirName := getRandomTerm() dirName := "srv/" + getRandomTerm()
addToIndex("/", dirName, true) addToIndex("/", dirName, true)
for j := 0; j < numFilesPerDir; j++ { for j := 0; j < numFilesPerDir; j++ {
fileName := "file-" + getRandomTerm() + getRandomExtension() fileName := "file-" + getRandomTerm() + getRandomExtension()