package files import ( "fmt" "os" "path/filepath" "runtime" "slices" "strings" "sync" "time" "github.com/gtsteffaniak/filebrowser/backend/cache" "github.com/gtsteffaniak/filebrowser/backend/logger" "github.com/gtsteffaniak/filebrowser/backend/settings" "github.com/gtsteffaniak/filebrowser/backend/utils" ) type Index struct { settings.Source Directories map[string]*FileInfo NumDirs uint64 NumFiles uint64 NumDeleted uint64 FilesChangedDuringIndexing bool currentSchedule int assessment string indexingTime int LastIndexed time.Time SmartModifier time.Duration mu sync.RWMutex scannerMu sync.Mutex } var ( indexes map[string]*Index indexesMutex sync.RWMutex RootPaths map[string]string ) func Initialize(source settings.Source) { indexesMutex.RLock() newIndex := Index{ Source: source, Directories: make(map[string]*FileInfo), } if RootPaths == nil { RootPaths = make(map[string]string) } RootPaths[source.Name] = source.Path indexes = make(map[string]*Index) indexes[newIndex.Source.Name] = &newIndex indexesMutex.RUnlock() if !newIndex.Source.Config.Disabled { time.Sleep(time.Second) logger.Info(fmt.Sprintf("initializing index: [%v]", newIndex.Source.Name)) newIndex.RunIndexing("/", false) go newIndex.setupIndexingScanners() } else { logger.Debug("indexing disabled for source: " + newIndex.Source.Name) } } // Define a function to recursively index files and directories func (idx *Index) indexDirectory(adjustedPath string, quick, recursive bool) error { realPath := strings.TrimRight(idx.Source.Path, "/") + adjustedPath // Open the directory dir, err := os.Open(realPath) if err != nil { idx.RemoveDirectory(adjustedPath) // Remove, must have been deleted return err } defer dir.Close() dirInfo, err := dir.Stat() if err != nil { return err } combinedPath := adjustedPath + "/" if adjustedPath == "/" { combinedPath = "/" } // get whats currently in cache idx.mu.RLock() cacheDirItems := []ItemInfo{} modChange := true // default to true cachedDir, exists := idx.Directories[adjustedPath] if exists && quick { modChange = dirInfo.ModTime() != cachedDir.ModTime cacheDirItems = cachedDir.Folders } idx.mu.RUnlock() // If the directory has not been modified since the last index, skip expensive readdir // recursively check cached dirs for mod time changes as well if !modChange && recursive { for _, item := range cacheDirItems { err = idx.indexDirectory(combinedPath+item.Name, quick, true) if err != nil { logger.Error(fmt.Sprintf("error indexing directory %v : %v", combinedPath+item.Name, err)) } } return nil } if quick { idx.mu.Lock() idx.FilesChangedDuringIndexing = true idx.mu.Unlock() } // Read directory contents files, err := dir.Readdir(-1) if err != nil { return err } var totalSize int64 fileInfos := []ItemInfo{} dirInfos := []ItemInfo{} // Process each file and directory in the current directory for _, file := range files { isHidden := isHidden(file, idx.Source.Path+combinedPath) isDir := file.IsDir() fullCombined := combinedPath + file.Name() if idx.shouldSkip(isDir, isHidden, fullCombined) { continue } itemInfo := &ItemInfo{ Name: file.Name(), ModTime: file.ModTime(), Hidden: isHidden, } // fix for .app files on macos which are technically directories, but we don't want to treat them as such if isDir && strings.HasSuffix(file.Name(), ".app") { isDir = false } if isDir { // skip non-indexable dirs. if file.Name() == "$RECYCLE.BIN" || file.Name() == "System Volume Information" { continue } dirPath := combinedPath + file.Name() if recursive { // Recursively index the subdirectory err = idx.indexDirectory(dirPath, quick, recursive) if err != nil { logger.Error(fmt.Sprintf("Failed to index directory %s: %v", dirPath, err)) continue } } realDirInfo, exists := idx.GetMetadataInfo(dirPath, true) if exists { itemInfo.Size = realDirInfo.Size } totalSize += itemInfo.Size itemInfo.Type = "directory" dirInfos = append(dirInfos, *itemInfo) idx.NumDirs++ } else { itemInfo.DetectType(fullCombined, false) itemInfo.Size = file.Size() fileInfos = append(fileInfos, *itemInfo) totalSize += itemInfo.Size idx.NumFiles++ } } if totalSize == 0 && idx.Source.Config.IgnoreZeroSizeFolders { return nil } // Create FileInfo for the current directory dirFileInfo := &FileInfo{ Path: adjustedPath, Files: fileInfos, Folders: dirInfos, } dirFileInfo.ItemInfo = ItemInfo{ Name: dirInfo.Name(), Type: "directory", Size: totalSize, ModTime: dirInfo.ModTime(), } dirFileInfo.SortItems() // Update the current directory metadata in the index idx.UpdateMetadata(dirFileInfo) return nil } func (idx *Index) makeIndexPath(subPath string) string { if strings.HasPrefix(subPath, "./") { subPath = strings.TrimPrefix(subPath, ".") } if idx.Source.Path == subPath || subPath == "." { return "/" } // clean path subPath = strings.TrimSuffix(subPath, "/") // remove index prefix adjustedPath := strings.TrimPrefix(subPath, idx.Source.Path) adjustedPath = strings.ReplaceAll(adjustedPath, "\\", "/") // remove trailing slash adjustedPath = strings.TrimSuffix(adjustedPath, "/") if !strings.HasPrefix(adjustedPath, "/") { adjustedPath = "/" + adjustedPath } return adjustedPath } func (idx *Index) recursiveUpdateDirSizes(childInfo *FileInfo, previousSize int64) { parentDir := utils.GetParentDirectoryPath(childInfo.Path) parentInfo, exists := idx.GetMetadataInfo(parentDir, true) if !exists || parentDir == "" { return } newSize := parentInfo.Size - previousSize + childInfo.Size parentInfo.Size += newSize idx.UpdateMetadata(parentInfo) idx.recursiveUpdateDirSizes(parentInfo, newSize) } func (idx *Index) GetRealPath(relativePath ...string) (string, bool, error) { combined := append([]string{idx.Source.Path}, relativePath...) joinedPath := filepath.Join(combined...) isDir, _ := cache.RealPath.Get(joinedPath + ":isdir").(bool) cached, ok := cache.RealPath.Get(joinedPath).(string) if ok && cached != "" { return cached, isDir, nil } // Convert relative path to absolute path absolutePath, err := filepath.Abs(joinedPath) if err != nil { return absolutePath, false, fmt.Errorf("could not get real path: %v, %s", joinedPath, err) } // Resolve symlinks and get the real path realPath, isDir, err := resolveSymlinks(absolutePath) if err == nil { cache.RealPath.Set(joinedPath, realPath) cache.RealPath.Set(joinedPath+":isdir", isDir) } return realPath, isDir, err } func (idx *Index) RefreshFileInfo(opts FileOptions) error { refreshOptions := FileOptions{ Path: opts.Path, IsDir: opts.IsDir, } if !refreshOptions.IsDir { refreshOptions.Path = idx.makeIndexPath(filepath.Dir(refreshOptions.Path)) refreshOptions.IsDir = true } else { refreshOptions.Path = idx.makeIndexPath(refreshOptions.Path) } err := idx.indexDirectory(refreshOptions.Path, false, false) if err != nil { return fmt.Errorf("file/folder does not exist to refresh data: %s", refreshOptions.Path) } file, exists := idx.GetMetadataInfo(refreshOptions.Path, true) if !exists { return fmt.Errorf("file/folder does not exist in metadata: %s", refreshOptions.Path) } current, firstExisted := idx.GetMetadataInfo(refreshOptions.Path, true) refreshParentInfo := firstExisted && current.Size != file.Size //utils.PrintStructFields(*file) result := idx.UpdateMetadata(file) if !result { return fmt.Errorf("file/folder does not exist in metadata: %s", refreshOptions.Path) } if !exists { return nil } if refreshParentInfo { idx.recursiveUpdateDirSizes(file, current.Size) } return nil } func isHidden(file os.FileInfo, srcPath string) bool { // Check if the file starts with a dot (common on Unix systems) if file.Name()[0] == '.' { return true } if runtime.GOOS == "windows" { return checkWindowsHidden(filepath.Join(srcPath, file.Name())) } // Default behavior for non-Windows systems return false } func (idx *Index) shouldSkip(isDir bool, isHidden bool, fullCombined string) bool { // check inclusions first if isDir && len(idx.Source.Config.Include.Folders) > 0 { if !slices.Contains(idx.Source.Config.Include.Folders, fullCombined) { return true } } if !isDir && len(idx.Source.Config.Include.Files) > 0 { if !slices.Contains(idx.Source.Config.Include.Files, fullCombined) { return true } } if !isDir && len(idx.Source.Config.Include.FileEndsWith) > 0 { shouldSkip := true for _, end := range idx.Source.Config.Include.FileEndsWith { if strings.HasSuffix(fullCombined, end) { shouldSkip = false break } } if shouldSkip { return true } } // check exclusions if isDir && slices.Contains(idx.Source.Config.Exclude.Folders, fullCombined) { return true } if !isDir && slices.Contains(idx.Source.Config.Exclude.Files, fullCombined) { return true } if idx.Source.Config.IgnoreHidden && isHidden { return true } if !isDir && len(idx.Source.Config.Exclude.FileEndsWith) > 0 { shouldSkip := false for _, end := range idx.Source.Config.Exclude.FileEndsWith { if strings.HasSuffix(fullCombined, end) { shouldSkip = true break } } return shouldSkip } return false }