Files previews: the cache is one database file, <data dir>/thumbs.db
bbolt instead of a tree of hash-named JPEGs: pure Go, so the binary stays static; one file per machine; an entry carries where the preview came from (engine, codec, size, when) beside the picture, and a count rides along so the tab can say what it holds without walking anything. A miss never creates the file, so a machine nobody browses with previews on never has one; a cache that cannot be opened is logged and previews are still made and served, just not kept. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
51c718e8e3
commit
e198471f8c
+182
-42
@@ -2,20 +2,51 @@ package thumbs
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// Cache keeps finished previews on disk, under Ham Lab's own data directory: one JPEG per
|
||||
// (path, size, mtime), so a file that changes gets a new preview and an unchanged one is
|
||||
// never decoded twice. A file ffmpeg could not decode is remembered the same way, with
|
||||
// ffmpeg's reason beside it, so a folder of odd files does not re-run ffmpeg on every
|
||||
// Cache keeps finished previews in one database file under Ham Lab's own data directory —
|
||||
// thumbs.db, a bbolt key/value store: pure Go, so the binary stays static, and one file
|
||||
// per machine rather than a tree of hash-named JPEGs. An entry is one file's preview keyed
|
||||
// by its path, size and mtime, so a file that changes gets a new preview and an unchanged
|
||||
// one is never decoded twice. A file ffmpeg could not decode is kept the same way, with
|
||||
// ffmpeg's reason and no picture, so a folder of odd files does not re-run ffmpeg on every
|
||||
// visit. Nothing expires: a preview is a few kilobytes, and only files somebody actually
|
||||
// scrolled past are ever made.
|
||||
type Cache struct{ Dir string }
|
||||
// scrolled past are ever made. The file is opened the first time something is stored, so a
|
||||
// machine nobody browses with previews on never has one.
|
||||
type Cache struct {
|
||||
Path string
|
||||
mu sync.Mutex
|
||||
db *bolt.DB
|
||||
err error
|
||||
}
|
||||
|
||||
// Entry is one cached preview, or the remembered reason there is none.
|
||||
type Entry struct {
|
||||
Path string `json:"path"`
|
||||
Engine string `json:"engine,omitempty"`
|
||||
Codec string `json:"codec,omitempty"`
|
||||
Width int `json:"width,omitempty"`
|
||||
Height int `json:"height,omitempty"`
|
||||
Made time.Time `json:"made"`
|
||||
Failure string `json:"failure,omitempty"` // set instead of JPEG when ffmpeg refused the file
|
||||
JPEG []byte `json:"-"`
|
||||
}
|
||||
|
||||
var (
|
||||
bucketThumbs = []byte("thumbs")
|
||||
bucketMeta = []byte("meta")
|
||||
keyCount = []byte("count")
|
||||
)
|
||||
|
||||
// Key identifies one file's preview. "v1" is the preview format: bump it to retire every
|
||||
// cached preview at once.
|
||||
@@ -24,55 +55,164 @@ func Key(path string, size, mtime int64) string {
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func (c Cache) file(key, ext string) string { return filepath.Join(c.Dir, key[:2], key+ext) }
|
||||
func NewCache(path string) *Cache { return &Cache{Path: path} }
|
||||
|
||||
// Get returns the preview, or the remembered reason it could not be made.
|
||||
func (c Cache) Get(key string) (jpeg []byte, failure string, found bool) {
|
||||
if len(key) < 2 {
|
||||
return nil, "", false
|
||||
// open the database once; an open that failed is not retried, and previews still get made
|
||||
// and served — they are simply not kept, and the log says why.
|
||||
func (c *Cache) open() (*bolt.DB, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.db != nil || c.err != nil {
|
||||
return c.db, c.err
|
||||
}
|
||||
if b, err := os.ReadFile(c.file(key, ".jpg")); err == nil && len(b) > 0 {
|
||||
return b, "", true
|
||||
db, err := bolt.Open(c.Path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return nil, err
|
||||
}
|
||||
if b, err := os.ReadFile(c.file(key, ".err")); err == nil {
|
||||
return nil, string(b), true
|
||||
if err := db.Update(func(tx *bolt.Tx) error {
|
||||
if _, err := tx.CreateBucketIfNotExists(bucketThumbs); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.CreateBucketIfNotExists(bucketMeta)
|
||||
return err
|
||||
}); err != nil {
|
||||
db.Close()
|
||||
c.err = err
|
||||
return nil, err
|
||||
}
|
||||
return nil, "", false
|
||||
c.db = db
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Put keeps a preview; PutFailure keeps the reason there is none.
|
||||
func (c Cache) Put(key string, jpeg []byte) error {
|
||||
if len(jpeg) == 0 {
|
||||
// Close releases the file. Previews after it are made but not kept.
|
||||
func (c *Cache) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.db == nil {
|
||||
return nil
|
||||
}
|
||||
err := c.db.Close()
|
||||
c.db, c.err = nil, errors.New("cache closed")
|
||||
return err
|
||||
}
|
||||
|
||||
// Get returns the entry for a key. A database that does not exist yet is a miss, not a reason
|
||||
// to make one.
|
||||
func (c *Cache) Get(key string) (Entry, bool, error) {
|
||||
c.mu.Lock()
|
||||
db := c.db
|
||||
c.mu.Unlock()
|
||||
if db == nil {
|
||||
if _, err := os.Stat(c.Path); err != nil {
|
||||
return Entry{}, false, nil
|
||||
}
|
||||
var err error
|
||||
if db, err = c.open(); err != nil {
|
||||
return Entry{}, false, err
|
||||
}
|
||||
}
|
||||
var e Entry
|
||||
found := false
|
||||
err := db.View(func(tx *bolt.Tx) error {
|
||||
v := tx.Bucket(bucketThumbs).Get([]byte(key))
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
e, err = decode(v) // copies: a value is only valid inside the transaction
|
||||
found = err == nil
|
||||
return err
|
||||
})
|
||||
return e, found, err
|
||||
}
|
||||
|
||||
// Put stores an entry, replacing any under the same key.
|
||||
func (c *Cache) Put(key string, e Entry) error {
|
||||
if len(e.JPEG) == 0 && e.Failure == "" {
|
||||
return errors.New("empty preview")
|
||||
}
|
||||
return c.write(c.file(key, ".jpg"), jpeg)
|
||||
}
|
||||
|
||||
func (c Cache) PutFailure(key, reason string) error {
|
||||
return c.write(c.file(key, ".err"), []byte(reason))
|
||||
}
|
||||
|
||||
// write lands the file whole or not at all: a reader never sees a half-written JPEG.
|
||||
func (c Cache) write(dst string, b []byte) error {
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
|
||||
return err
|
||||
if e.Made.IsZero() {
|
||||
e.Made = time.Now()
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dst), ".tmp-*")
|
||||
db, err := c.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tmp.Write(b); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmp.Name())
|
||||
v, err := encode(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return err
|
||||
return db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(bucketThumbs)
|
||||
k := []byte(key)
|
||||
fresh := b.Get(k) == nil
|
||||
if err := b.Put(k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if fresh {
|
||||
m := tx.Bucket(bucketMeta)
|
||||
return m.Put(keyCount, be64(count(m)+1))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Stats is how many previews are kept and how big the file is.
|
||||
func (c *Cache) Stats() (entries int64, bytes int64) {
|
||||
st, err := os.Stat(c.Path)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), dst); err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return err
|
||||
bytes = st.Size()
|
||||
db, err := c.open()
|
||||
if err != nil {
|
||||
return 0, bytes
|
||||
}
|
||||
return nil
|
||||
_ = db.View(func(tx *bolt.Tx) error { entries = count(tx.Bucket(bucketMeta)); return nil })
|
||||
return entries, bytes
|
||||
}
|
||||
|
||||
func count(m *bolt.Bucket) int64 {
|
||||
if v := m.Get(keyCount); len(v) == 8 {
|
||||
return int64(binary.BigEndian.Uint64(v))
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func be64(n int64) []byte {
|
||||
b := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(b, uint64(n))
|
||||
return b
|
||||
}
|
||||
|
||||
// A value is a 4-byte header length, the header as JSON, then the JPEG bytes.
|
||||
func encode(e Entry) ([]byte, error) {
|
||||
hdr, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v := make([]byte, 4+len(hdr)+len(e.JPEG))
|
||||
binary.BigEndian.PutUint32(v, uint32(len(hdr)))
|
||||
copy(v[4:], hdr)
|
||||
copy(v[4+len(hdr):], e.JPEG)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func decode(v []byte) (Entry, error) {
|
||||
if len(v) < 4 {
|
||||
return Entry{}, errors.New("short value")
|
||||
}
|
||||
n := int(binary.BigEndian.Uint32(v))
|
||||
if 4+n > len(v) {
|
||||
return Entry{}, errors.New("bad header length")
|
||||
}
|
||||
var e Entry
|
||||
if err := json.Unmarshal(v[4:4+n], &e); err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
if rest := v[4+n:]; len(rest) > 0 {
|
||||
e.JPEG = append([]byte(nil), rest...)
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -107,30 +107,54 @@ func TestAttempts(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
c := Cache{Dir: t.TempDir()}
|
||||
c := NewCache(filepath.Join(t.TempDir(), "thumbs.db"))
|
||||
k := Key("/a", 1, 2)
|
||||
if _, _, found := c.Get(k); found {
|
||||
t.Fatal("empty cache found something")
|
||||
if _, found, err := c.Get(k); found || err != nil {
|
||||
t.Fatal("an empty cache found something, or complained")
|
||||
}
|
||||
if err := c.Put(k, []byte("jpeg")); err != nil {
|
||||
if _, err := os.Stat(c.Path); err == nil {
|
||||
t.Fatal("a miss must not create the database")
|
||||
}
|
||||
if n, b := c.Stats(); n != 0 || b != 0 {
|
||||
t.Fatal("stats of nothing")
|
||||
}
|
||||
if err := c.Put(k, Entry{Path: "/a", Engine: "cpu", Codec: "png", Width: 640, Height: 360, JPEG: []byte("jpeg")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _, found := c.Get(k); !found || string(b) != "jpeg" {
|
||||
t.Fatal("put then get")
|
||||
e, found, err := c.Get(k)
|
||||
if err != nil || !found || string(e.JPEG) != "jpeg" || e.Engine != "cpu" || e.Width != 640 || e.Path != "/a" || e.Made.IsZero() {
|
||||
t.Fatalf("put then get: %+v %v %v", e, found, err)
|
||||
}
|
||||
k2 := Key("/b", 1, 2)
|
||||
if err := c.PutFailure(k2, "no video stream"); err != nil {
|
||||
if err := c.Put(k2, Entry{Path: "/b", Failure: "no video stream"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, why, found := c.Get(k2); !found || b != nil || why != "no video stream" {
|
||||
if e, found, _ := c.Get(k2); !found || e.JPEG != nil || e.Failure != "no video stream" {
|
||||
t.Fatal("a failure is remembered with its reason")
|
||||
}
|
||||
if err := c.Put(k, nil); err == nil {
|
||||
if err := c.Put(k, Entry{}); err == nil {
|
||||
t.Fatal("an empty preview must not be kept")
|
||||
}
|
||||
if left, _ := filepath.Glob(filepath.Join(c.Dir, "*", ".tmp-*")); len(left) != 0 {
|
||||
t.Fatal("temp files left behind")
|
||||
// replacing an entry does not count it twice
|
||||
if err := c.Put(k, Entry{Path: "/a", JPEG: []byte("jpeg2")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, b := c.Stats()
|
||||
if n != 2 || b == 0 {
|
||||
t.Fatalf("stats = %d entries, %d bytes", n, b)
|
||||
}
|
||||
// survives a reopen
|
||||
if err := c.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c2 := NewCache(c.Path)
|
||||
if e, found, _ := c2.Get(k); !found || string(e.JPEG) != "jpeg2" {
|
||||
t.Fatal("not there after reopening")
|
||||
}
|
||||
if n, _ := c2.Stats(); n != 2 {
|
||||
t.Fatal("count lost on reopen")
|
||||
}
|
||||
c2.Close()
|
||||
}
|
||||
|
||||
// The real thing, when ffmpeg is here: a PNG and a generated video each come back as a
|
||||
|
||||
Reference in New Issue
Block a user