diff --git a/go.mod b/go.mod index 7c03e2c..35dec24 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.0 require ( github.com/coder/websocket v1.8.15 github.com/creack/pty v1.1.24 + go.etcd.io/bbolt v1.5.0 golang.org/x/crypto v0.57.0 golang.org/x/term v0.46.0 ) diff --git a/go.sum b/go.sum index f1c4538..64a304c 100644 --- a/go.sum +++ b/go.sum @@ -2,9 +2,21 @@ github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNU github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/thumbs.go b/internal/api/thumbs.go index bd606da..996e3b5 100644 --- a/internal/api/thumbs.go +++ b/internal/api/thumbs.go @@ -1,7 +1,7 @@ package api // Previews for the Files tab: GET /api/files/thumb hands back one small JPEG of an image or -// a video, made by internal/thumbs and kept under /thumbs/. The work is bounded +// a video, made by internal/thumbs and kept in /thumbs.db. The work is bounded // two ways — a handful of ffmpegs at once, and one per file however many rows ask for it — // because a folder of a thousand photos asks for every row the moment previews go on. // @@ -32,7 +32,7 @@ const thumbTimeout = 30 * time.Second const capsTTL = 10 * time.Minute // how long a probe is believed; a refresh after an install or a group change does not wait type thumbSvc struct { - cache thumbs.Cache + cache *thumbs.Cache self string sem chan struct{} mu sync.Mutex @@ -61,7 +61,7 @@ func newThumbSvc(dataDir, self string) *thumbSvc { if n > 4 { n = 4 } - return &thumbSvc{cache: thumbs.Cache{Dir: filepath.Join(dataDir, "thumbs")}, self: self, sem: make(chan struct{}, n), caps: map[bool]*capsEntry{}, calls: map[string]*thumbCall{}} + return &thumbSvc{cache: thumbs.NewCache(filepath.Join(dataDir, "thumbs.db")), self: self, sem: make(chan struct{}, n), caps: map[bool]*capsEntry{}, calls: map[string]*thumbCall{}} } // capsFor is what previews can be made with, probed once and kept for a while. @@ -208,7 +208,9 @@ func (s *Server) filesThumb(w http.ResponseWriter, r *http.Request) { var de *thumbs.DecodeError switch { case errors.As(err, &de): - _ = s.thumbs.cache.PutFailure(key, de.Msg) + if err := s.thumbs.cache.Put(key, thumbs.Entry{Path: p, Failure: de.Msg}); err != nil { + log.Printf("thumbs: cache: %v", err) + } log.Printf("thumbs: %s: %s", p, de.Msg) jsonErr(w, 422, de.Msg) case errors.Is(err, context.DeadlineExceeded): @@ -221,21 +223,26 @@ func (s *Server) filesThumb(w http.ResponseWriter, r *http.Request) { } return } - if err := s.thumbs.cache.Put(key, res.JPEG); err != nil { - log.Printf("thumbs: cache %s: %v", key[:8], err) + if err := s.thumbs.cache.Put(key, thumbs.Entry{Path: p, Engine: res.Engine, Codec: res.Codec, Width: res.Width, Height: res.Height, JPEG: res.JPEG}); err != nil { + log.Printf("thumbs: cache: %v", err) } writeThumb(w, key, res.JPEG, res.Engine) } +// serveCachedThumb answers from the cache when it can. A cache that cannot be read is a +// miss: the preview is made again rather than refused. func (s *Server) serveCachedThumb(w http.ResponseWriter, key string) bool { - data, why, found := s.thumbs.cache.Get(key) + e, found, err := s.thumbs.cache.Get(key) + if err != nil { + log.Printf("thumbs: cache: %v", err) + } switch { case !found: return false - case data == nil: - jsonErr(w, 422, why) + case e.Failure != "": + jsonErr(w, 422, e.Failure) default: - writeThumb(w, key, data, "cache") + writeThumb(w, key, e.JPEG, "cache ("+e.Engine+")") } return true } @@ -271,7 +278,13 @@ type thumbStatus struct { Install *thumbInstall `json:"install,omitempty"` // ffmpeg is missing and this can put it on GPUFix bool `json:"gpu_fix,omitempty"` // a render node is there but closed to the daemon's user, and this can open it User string `json:"user"` - CacheDir string `json:"cache_dir"` + Cache thumbCache `json:"cache"` +} + +type thumbCache struct { + Path string `json:"path"` + Entries int64 `json:"entries"` + Bytes int64 `json:"bytes"` } type thumbInstall struct { @@ -287,7 +300,8 @@ func (s *Server) thumbsStatus(w http.ResponseWriter, r *http.Request) { return } caps, err := s.thumbs.capsFor(r.Context(), root, r.URL.Query().Get("refresh") == "1") - st := thumbStatus{GPUs: []thumbs.GPU{}, User: s.user, CacheDir: s.thumbs.cache.Dir} + st := thumbStatus{GPUs: []thumbs.GPU{}, User: s.user, Cache: thumbCache{Path: s.thumbs.cache.Path}} + st.Cache.Entries, st.Cache.Bytes = s.thumbs.cache.Stats() if root { st.User = "root" } diff --git a/internal/thumbs/cache.go b/internal/thumbs/cache.go index f90c8f8..d3d9998 100644 --- a/internal/thumbs/cache.go +++ b/internal/thumbs/cache.go @@ -2,20 +2,51 @@ package thumbs import ( "crypto/sha256" + "encoding/binary" "encoding/hex" + "encoding/json" "errors" "os" - "path/filepath" "strconv" + "sync" + "time" + + bolt "go.etcd.io/bbolt" ) -// Cache keeps finished previews on disk, under Ham Lab's own data directory: one JPEG per -// (path, size, mtime), so a file that changes gets a new preview and an unchanged one is -// never decoded twice. A file ffmpeg could not decode is remembered the same way, with -// ffmpeg's reason beside it, so a folder of odd files does not re-run ffmpeg on every +// Cache keeps finished previews in one database file under Ham Lab's own data directory — +// thumbs.db, a bbolt key/value store: pure Go, so the binary stays static, and one file +// per machine rather than a tree of hash-named JPEGs. An entry is one file's preview keyed +// by its path, size and mtime, so a file that changes gets a new preview and an unchanged +// one is never decoded twice. A file ffmpeg could not decode is kept the same way, with +// ffmpeg's reason and no picture, so a folder of odd files does not re-run ffmpeg on every // visit. Nothing expires: a preview is a few kilobytes, and only files somebody actually -// scrolled past are ever made. -type Cache struct{ Dir string } +// scrolled past are ever made. The file is opened the first time something is stored, so a +// machine nobody browses with previews on never has one. +type Cache struct { + Path string + mu sync.Mutex + db *bolt.DB + err error +} + +// Entry is one cached preview, or the remembered reason there is none. +type Entry struct { + Path string `json:"path"` + Engine string `json:"engine,omitempty"` + Codec string `json:"codec,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + Made time.Time `json:"made"` + Failure string `json:"failure,omitempty"` // set instead of JPEG when ffmpeg refused the file + JPEG []byte `json:"-"` +} + +var ( + bucketThumbs = []byte("thumbs") + bucketMeta = []byte("meta") + keyCount = []byte("count") +) // Key identifies one file's preview. "v1" is the preview format: bump it to retire every // cached preview at once. @@ -24,55 +55,164 @@ func Key(path string, size, mtime int64) string { return hex.EncodeToString(h[:]) } -func (c Cache) file(key, ext string) string { return filepath.Join(c.Dir, key[:2], key+ext) } +func NewCache(path string) *Cache { return &Cache{Path: path} } -// Get returns the preview, or the remembered reason it could not be made. -func (c Cache) Get(key string) (jpeg []byte, failure string, found bool) { - if len(key) < 2 { - return nil, "", false +// open the database once; an open that failed is not retried, and previews still get made +// and served — they are simply not kept, and the log says why. +func (c *Cache) open() (*bolt.DB, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.db != nil || c.err != nil { + return c.db, c.err } - if b, err := os.ReadFile(c.file(key, ".jpg")); err == nil && len(b) > 0 { - return b, "", true + db, err := bolt.Open(c.Path, 0o600, &bolt.Options{Timeout: 2 * time.Second}) + if err != nil { + c.err = err + return nil, err } - if b, err := os.ReadFile(c.file(key, ".err")); err == nil { - return nil, string(b), true + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists(bucketThumbs); err != nil { + return err + } + _, err := tx.CreateBucketIfNotExists(bucketMeta) + return err + }); err != nil { + db.Close() + c.err = err + return nil, err } - return nil, "", false + c.db = db + return db, nil } -// Put keeps a preview; PutFailure keeps the reason there is none. -func (c Cache) Put(key string, jpeg []byte) error { - if len(jpeg) == 0 { +// Close releases the file. Previews after it are made but not kept. +func (c *Cache) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.db == nil { + return nil + } + err := c.db.Close() + c.db, c.err = nil, errors.New("cache closed") + return err +} + +// Get returns the entry for a key. A database that does not exist yet is a miss, not a reason +// to make one. +func (c *Cache) Get(key string) (Entry, bool, error) { + c.mu.Lock() + db := c.db + c.mu.Unlock() + if db == nil { + if _, err := os.Stat(c.Path); err != nil { + return Entry{}, false, nil + } + var err error + if db, err = c.open(); err != nil { + return Entry{}, false, err + } + } + var e Entry + found := false + err := db.View(func(tx *bolt.Tx) error { + v := tx.Bucket(bucketThumbs).Get([]byte(key)) + if v == nil { + return nil + } + var err error + e, err = decode(v) // copies: a value is only valid inside the transaction + found = err == nil + return err + }) + return e, found, err +} + +// Put stores an entry, replacing any under the same key. +func (c *Cache) Put(key string, e Entry) error { + if len(e.JPEG) == 0 && e.Failure == "" { return errors.New("empty preview") } - return c.write(c.file(key, ".jpg"), jpeg) -} - -func (c Cache) PutFailure(key, reason string) error { - return c.write(c.file(key, ".err"), []byte(reason)) -} - -// write lands the file whole or not at all: a reader never sees a half-written JPEG. -func (c Cache) write(dst string, b []byte) error { - if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { - return err + if e.Made.IsZero() { + e.Made = time.Now() } - tmp, err := os.CreateTemp(filepath.Dir(dst), ".tmp-*") + db, err := c.open() if err != nil { return err } - if _, err := tmp.Write(b); err != nil { - tmp.Close() - os.Remove(tmp.Name()) + v, err := encode(e) + if err != nil { return err } - if err := tmp.Close(); err != nil { - os.Remove(tmp.Name()) - return err + return db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket(bucketThumbs) + k := []byte(key) + fresh := b.Get(k) == nil + if err := b.Put(k, v); err != nil { + return err + } + if fresh { + m := tx.Bucket(bucketMeta) + return m.Put(keyCount, be64(count(m)+1)) + } + return nil + }) +} + +// Stats is how many previews are kept and how big the file is. +func (c *Cache) Stats() (entries int64, bytes int64) { + st, err := os.Stat(c.Path) + if err != nil { + return 0, 0 } - if err := os.Rename(tmp.Name(), dst); err != nil { - os.Remove(tmp.Name()) - return err + bytes = st.Size() + db, err := c.open() + if err != nil { + return 0, bytes } - return nil + _ = db.View(func(tx *bolt.Tx) error { entries = count(tx.Bucket(bucketMeta)); return nil }) + return entries, bytes +} + +func count(m *bolt.Bucket) int64 { + if v := m.Get(keyCount); len(v) == 8 { + return int64(binary.BigEndian.Uint64(v)) + } + return 0 +} + +func be64(n int64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(n)) + return b +} + +// A value is a 4-byte header length, the header as JSON, then the JPEG bytes. +func encode(e Entry) ([]byte, error) { + hdr, err := json.Marshal(e) + if err != nil { + return nil, err + } + v := make([]byte, 4+len(hdr)+len(e.JPEG)) + binary.BigEndian.PutUint32(v, uint32(len(hdr))) + copy(v[4:], hdr) + copy(v[4+len(hdr):], e.JPEG) + return v, nil +} + +func decode(v []byte) (Entry, error) { + if len(v) < 4 { + return Entry{}, errors.New("short value") + } + n := int(binary.BigEndian.Uint32(v)) + if 4+n > len(v) { + return Entry{}, errors.New("bad header length") + } + var e Entry + if err := json.Unmarshal(v[4:4+n], &e); err != nil { + return Entry{}, err + } + if rest := v[4+n:]; len(rest) > 0 { + e.JPEG = append([]byte(nil), rest...) + } + return e, nil } diff --git a/internal/thumbs/thumbs_test.go b/internal/thumbs/thumbs_test.go index 9941725..3f176ca 100644 --- a/internal/thumbs/thumbs_test.go +++ b/internal/thumbs/thumbs_test.go @@ -107,30 +107,54 @@ func TestAttempts(t *testing.T) { } func TestCache(t *testing.T) { - c := Cache{Dir: t.TempDir()} + c := NewCache(filepath.Join(t.TempDir(), "thumbs.db")) k := Key("/a", 1, 2) - if _, _, found := c.Get(k); found { - t.Fatal("empty cache found something") + if _, found, err := c.Get(k); found || err != nil { + t.Fatal("an empty cache found something, or complained") } - if err := c.Put(k, []byte("jpeg")); err != nil { + if _, err := os.Stat(c.Path); err == nil { + t.Fatal("a miss must not create the database") + } + if n, b := c.Stats(); n != 0 || b != 0 { + t.Fatal("stats of nothing") + } + if err := c.Put(k, Entry{Path: "/a", Engine: "cpu", Codec: "png", Width: 640, Height: 360, JPEG: []byte("jpeg")}); err != nil { t.Fatal(err) } - if b, _, found := c.Get(k); !found || string(b) != "jpeg" { - t.Fatal("put then get") + e, found, err := c.Get(k) + if err != nil || !found || string(e.JPEG) != "jpeg" || e.Engine != "cpu" || e.Width != 640 || e.Path != "/a" || e.Made.IsZero() { + t.Fatalf("put then get: %+v %v %v", e, found, err) } k2 := Key("/b", 1, 2) - if err := c.PutFailure(k2, "no video stream"); err != nil { + if err := c.Put(k2, Entry{Path: "/b", Failure: "no video stream"}); err != nil { t.Fatal(err) } - if b, why, found := c.Get(k2); !found || b != nil || why != "no video stream" { + if e, found, _ := c.Get(k2); !found || e.JPEG != nil || e.Failure != "no video stream" { t.Fatal("a failure is remembered with its reason") } - if err := c.Put(k, nil); err == nil { + if err := c.Put(k, Entry{}); err == nil { t.Fatal("an empty preview must not be kept") } - if left, _ := filepath.Glob(filepath.Join(c.Dir, "*", ".tmp-*")); len(left) != 0 { - t.Fatal("temp files left behind") + // replacing an entry does not count it twice + if err := c.Put(k, Entry{Path: "/a", JPEG: []byte("jpeg2")}); err != nil { + t.Fatal(err) } + n, b := c.Stats() + if n != 2 || b == 0 { + t.Fatalf("stats = %d entries, %d bytes", n, b) + } + // survives a reopen + if err := c.Close(); err != nil { + t.Fatal(err) + } + c2 := NewCache(c.Path) + if e, found, _ := c2.Get(k); !found || string(e.JPEG) != "jpeg2" { + t.Fatal("not there after reopening") + } + if n, _ := c2.Stats(); n != 2 { + t.Fatal("count lost on reopen") + } + c2.Close() } // The real thing, when ffmpeg is here: a PNG and a generated video each come back as a diff --git a/web/src/Files.tsx b/web/src/Files.tsx index d35e746..6fdf95f 100644 --- a/web/src/Files.tsx +++ b/web/src/Files.tsx @@ -192,7 +192,7 @@ export default function Files({ onOpenInTerminal }: { onOpenInTerminal?: (cmd: s const [thumbs, setThumbs] = useState(null) const [listEl, setListEl] = useState(null) const near = useMemo(() => (listEl && previews ? new Near(listEl) : null), [listEl, previews]) - const readThumbs = useCallback((refresh = false) => { api.thumbStatus(sudo, refresh).then(setThumbs).catch((e) => setThumbs({ ok: false, engine: '', summary: '', gpus: [], error: e instanceof Error ? e.message : 'failed', user: '', cache_dir: '' })) }, [sudo]) + const readThumbs = useCallback((refresh = false) => { api.thumbStatus(sudo, refresh).then(setThumbs).catch((e) => setThumbs({ ok: false, engine: '', summary: '', gpus: [], error: e instanceof Error ? e.message : 'failed', user: '', cache: { path: '', entries: 0, bytes: 0 } })) }, [sudo]) useEffect(() => { try { localStorage.setItem(PREVIEWS_KEY, previews ? '1' : '0') } catch { /* private mode */ } }, [previews]) useEffect(() => { if (previews) readThumbs(); else setThumbs(null) }, [previews, readThumbs]) @@ -279,7 +279,7 @@ export default function Files({ onOpenInTerminal }: { onOpenInTerminal?: (cmd: s
- + setQ(e.target.value)} /> diff --git a/web/src/api.ts b/web/src/api.ts index 814bcb7..52012d6 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -252,7 +252,7 @@ export interface ARCView { detail: ARCDetail; live: ARCLive | null } export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean; preview?: 'image' | 'video' } export interface ThumbGPU { api: string; device: string; name?: string; codecs?: string[]; reason?: string; permission?: boolean } // How previews get made on the machine: ok with an engine, or why not and what would fix it. -export interface ThumbStatus { ok: boolean; engine: 'gpu' | 'cpu' | ''; summary: string; ffmpeg?: string; gpus: ThumbGPU[]; unusable?: ThumbGPU[]; error?: string; install?: { pm: string; package: string }; gpu_fix?: boolean; user: string; cache_dir: string } +export interface ThumbStatus { ok: boolean; engine: 'gpu' | 'cpu' | ''; summary: string; ffmpeg?: string; gpus: ThumbGPU[]; unusable?: ThumbGPU[]; error?: string; install?: { pm: string; package: string }; gpu_fix?: boolean; user: string; cache: { path: string; entries: number; bytes: number } } export interface FileInfo extends FileEntry { path: string; text: boolean } const q = (path: string, sudo: boolean, extra = '') => `path=${encodeURIComponent(path)}${sudo ? '&sudo=1' : ''}${extra}`