Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e5d3fb2afd | ||
|
|
e198471f8c |
@@ -652,16 +652,23 @@ something long in one, switch away and back, refresh — it's all still there.
|
|||||||
0.7 s on the CPU; the 8K Osmo HEVC 1.0 s against 2.9 s. `grid`'s two GPUs are listed in
|
0.7 s on the CPU; the 8K Osmo HEVC 1.0 s against 2.9 s. `grid`'s two GPUs are listed in
|
||||||
node order and both decode the same set (h264, hevc, vp9, mjpeg, mpeg2 — **no AV1 on
|
node order and both decode the same set (h264, hevc, vp9, mjpeg, mpeg2 — **no AV1 on
|
||||||
either**, per vainfo, so AV1 is dav1d on the CPU).
|
either**, per vainfo, so AV1 is dav1d on the CPU).
|
||||||
- **Cached under the machine's data dir**, `<data dir>/thumbs/<xx>/<sha256>.jpg`, keyed by
|
- **Cached in one database file under the machine's data dir**, `<data dir>/thumbs.db`
|
||||||
path, size and mtime — a changed file gets a new preview, an unchanged one is never
|
(bbolt — pure Go, so the binary stays static; v0.1.0-202, the first build kept them as
|
||||||
decoded twice, and the URL carries `v=<mtime>-<size>` so the browser's own cache is
|
a tree of hash-named JPEGs under `thumbs/`, replaced the same day when Mitchell asked
|
||||||
right too (7 days, ETag, 304). A file ffmpeg could not decode is remembered beside the
|
where they were kept: "stored in hamlab, maybe under a DB, up to you"). An entry is
|
||||||
JPEGs as `.err` with ffmpeg's reason, answered as 422, and retried only when the file
|
keyed by path, size and mtime — a changed file gets a new preview, an unchanged one is
|
||||||
changes; a timeout is not remembered. Nothing expires: a preview is a few KB, and only
|
never decoded twice — and carries the engine, codec, source size and time beside the
|
||||||
|
picture; a count rides in the file so the previews button's tooltip can say "N kept
|
||||||
|
(M MB)" without walking anything. The URL carries `v=<mtime>-<size>` so the browser's
|
||||||
|
own cache is right too (7 days, ETag, 304). A file ffmpeg could not decode is kept with
|
||||||
|
ffmpeg's reason and no picture, answered as 422, retried only when the file changes; a
|
||||||
|
timeout is not kept. A miss never creates the file, so a machine nobody browses with
|
||||||
|
previews on never has one; a cache that cannot be opened is logged and previews are
|
||||||
|
still made and served, just not kept. Nothing expires: a preview is a few KB, and only
|
||||||
files somebody scrolled past are ever made. At most 2–4 ffmpegs at once per machine
|
files somebody scrolled past are ever made. At most 2–4 ffmpegs at once per machine
|
||||||
(half the cores, clamped), one per file however many rows ask, and a preview's work
|
(half the cores, clamped), one per file however many rows ask, and a preview's work
|
||||||
runs on a clock of its own so the row that asked first leaving does not cancel it for
|
runs on a clock of its own so the row that asked first leaving does not cancel it for
|
||||||
the rest. Each ffmpeg runs under `nice -n 10`. `hamlet`'s backup job skips the folder.
|
the rest. Each ffmpeg runs under `nice -n 10`. `hamlet`'s backup job skips the file.
|
||||||
- **Root mode makes them as root** (`hamlab thumb`, the sibling of `hamlab fs`, caps on
|
- **Root mode makes them as root** (`hamlab thumb`, the sibling of `hamlab fs`, caps on
|
||||||
stdin, JSON out), so a file only root can read gets a preview, and the probe runs as
|
stdin, JSON out), so a file only root can read gets a preview, and the probe runs as
|
||||||
root too — root opens every render node whatever groups the daemon has. Which matters:
|
root too — root opens every render node whatever groups the daemon has. Which matters:
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ go 1.26.0
|
|||||||
require (
|
require (
|
||||||
github.com/coder/websocket v1.8.15
|
github.com/coder/websocket v1.8.15
|
||||||
github.com/creack/pty v1.1.24
|
github.com/creack/pty v1.1.24
|
||||||
|
go.etcd.io/bbolt v1.5.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
golang.org/x/term v0.46.0
|
golang.org/x/term v0.46.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -2,9 +2,21 @@ github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNU
|
|||||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
|
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||||
|
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
|
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||||
|
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
||||||
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
+26
-12
@@ -1,7 +1,7 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
// Previews for the Files tab: GET /api/files/thumb hands back one small JPEG of an image or
|
// Previews for the Files tab: GET /api/files/thumb hands back one small JPEG of an image or
|
||||||
// a video, made by internal/thumbs and kept under <data dir>/thumbs/. The work is bounded
|
// a video, made by internal/thumbs and kept in <data dir>/thumbs.db. The work is bounded
|
||||||
// two ways — a handful of ffmpegs at once, and one per file however many rows ask for it —
|
// two ways — a handful of ffmpegs at once, and one per file however many rows ask for it —
|
||||||
// because a folder of a thousand photos asks for every row the moment previews go on.
|
// because a folder of a thousand photos asks for every row the moment previews go on.
|
||||||
//
|
//
|
||||||
@@ -32,7 +32,7 @@ const thumbTimeout = 30 * time.Second
|
|||||||
const capsTTL = 10 * time.Minute // how long a probe is believed; a refresh after an install or a group change does not wait
|
const capsTTL = 10 * time.Minute // how long a probe is believed; a refresh after an install or a group change does not wait
|
||||||
|
|
||||||
type thumbSvc struct {
|
type thumbSvc struct {
|
||||||
cache thumbs.Cache
|
cache *thumbs.Cache
|
||||||
self string
|
self string
|
||||||
sem chan struct{}
|
sem chan struct{}
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
@@ -61,7 +61,7 @@ func newThumbSvc(dataDir, self string) *thumbSvc {
|
|||||||
if n > 4 {
|
if n > 4 {
|
||||||
n = 4
|
n = 4
|
||||||
}
|
}
|
||||||
return &thumbSvc{cache: thumbs.Cache{Dir: filepath.Join(dataDir, "thumbs")}, self: self, sem: make(chan struct{}, n), caps: map[bool]*capsEntry{}, calls: map[string]*thumbCall{}}
|
return &thumbSvc{cache: thumbs.NewCache(filepath.Join(dataDir, "thumbs.db")), self: self, sem: make(chan struct{}, n), caps: map[bool]*capsEntry{}, calls: map[string]*thumbCall{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// capsFor is what previews can be made with, probed once and kept for a while.
|
// capsFor is what previews can be made with, probed once and kept for a while.
|
||||||
@@ -208,7 +208,9 @@ func (s *Server) filesThumb(w http.ResponseWriter, r *http.Request) {
|
|||||||
var de *thumbs.DecodeError
|
var de *thumbs.DecodeError
|
||||||
switch {
|
switch {
|
||||||
case errors.As(err, &de):
|
case errors.As(err, &de):
|
||||||
_ = s.thumbs.cache.PutFailure(key, de.Msg)
|
if err := s.thumbs.cache.Put(key, thumbs.Entry{Path: p, Failure: de.Msg}); err != nil {
|
||||||
|
log.Printf("thumbs: cache: %v", err)
|
||||||
|
}
|
||||||
log.Printf("thumbs: %s: %s", p, de.Msg)
|
log.Printf("thumbs: %s: %s", p, de.Msg)
|
||||||
jsonErr(w, 422, de.Msg)
|
jsonErr(w, 422, de.Msg)
|
||||||
case errors.Is(err, context.DeadlineExceeded):
|
case errors.Is(err, context.DeadlineExceeded):
|
||||||
@@ -221,21 +223,26 @@ func (s *Server) filesThumb(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := s.thumbs.cache.Put(key, res.JPEG); err != nil {
|
if err := s.thumbs.cache.Put(key, thumbs.Entry{Path: p, Engine: res.Engine, Codec: res.Codec, Width: res.Width, Height: res.Height, JPEG: res.JPEG}); err != nil {
|
||||||
log.Printf("thumbs: cache %s: %v", key[:8], err)
|
log.Printf("thumbs: cache: %v", err)
|
||||||
}
|
}
|
||||||
writeThumb(w, key, res.JPEG, res.Engine)
|
writeThumb(w, key, res.JPEG, res.Engine)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// serveCachedThumb answers from the cache when it can. A cache that cannot be read is a
|
||||||
|
// miss: the preview is made again rather than refused.
|
||||||
func (s *Server) serveCachedThumb(w http.ResponseWriter, key string) bool {
|
func (s *Server) serveCachedThumb(w http.ResponseWriter, key string) bool {
|
||||||
data, why, found := s.thumbs.cache.Get(key)
|
e, found, err := s.thumbs.cache.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("thumbs: cache: %v", err)
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case !found:
|
case !found:
|
||||||
return false
|
return false
|
||||||
case data == nil:
|
case e.Failure != "":
|
||||||
jsonErr(w, 422, why)
|
jsonErr(w, 422, e.Failure)
|
||||||
default:
|
default:
|
||||||
writeThumb(w, key, data, "cache")
|
writeThumb(w, key, e.JPEG, "cache ("+e.Engine+")")
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -271,7 +278,13 @@ type thumbStatus struct {
|
|||||||
Install *thumbInstall `json:"install,omitempty"` // ffmpeg is missing and this can put it on
|
Install *thumbInstall `json:"install,omitempty"` // ffmpeg is missing and this can put it on
|
||||||
GPUFix bool `json:"gpu_fix,omitempty"` // a render node is there but closed to the daemon's user, and this can open it
|
GPUFix bool `json:"gpu_fix,omitempty"` // a render node is there but closed to the daemon's user, and this can open it
|
||||||
User string `json:"user"`
|
User string `json:"user"`
|
||||||
CacheDir string `json:"cache_dir"`
|
Cache thumbCache `json:"cache"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type thumbCache struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Entries int64 `json:"entries"`
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type thumbInstall struct {
|
type thumbInstall struct {
|
||||||
@@ -287,7 +300,8 @@ func (s *Server) thumbsStatus(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
caps, err := s.thumbs.capsFor(r.Context(), root, r.URL.Query().Get("refresh") == "1")
|
caps, err := s.thumbs.capsFor(r.Context(), root, r.URL.Query().Get("refresh") == "1")
|
||||||
st := thumbStatus{GPUs: []thumbs.GPU{}, User: s.user, CacheDir: s.thumbs.cache.Dir}
|
st := thumbStatus{GPUs: []thumbs.GPU{}, User: s.user, Cache: thumbCache{Path: s.thumbs.cache.Path}}
|
||||||
|
st.Cache.Entries, st.Cache.Bytes = s.thumbs.cache.Stats()
|
||||||
if root {
|
if root {
|
||||||
st.User = "root"
|
st.User = "root"
|
||||||
}
|
}
|
||||||
|
|||||||
+182
-42
@@ -2,20 +2,51 @@ package thumbs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/binary"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
bolt "go.etcd.io/bbolt"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Cache keeps finished previews on disk, under Ham Lab's own data directory: one JPEG per
|
// Cache keeps finished previews in one database file under Ham Lab's own data directory —
|
||||||
// (path, size, mtime), so a file that changes gets a new preview and an unchanged one is
|
// thumbs.db, a bbolt key/value store: pure Go, so the binary stays static, and one file
|
||||||
// never decoded twice. A file ffmpeg could not decode is remembered the same way, with
|
// per machine rather than a tree of hash-named JPEGs. An entry is one file's preview keyed
|
||||||
// ffmpeg's reason beside it, so a folder of odd files does not re-run ffmpeg on every
|
// by its path, size and mtime, so a file that changes gets a new preview and an unchanged
|
||||||
|
// one is never decoded twice. A file ffmpeg could not decode is kept the same way, with
|
||||||
|
// ffmpeg's reason and no picture, so a folder of odd files does not re-run ffmpeg on every
|
||||||
// visit. Nothing expires: a preview is a few kilobytes, and only files somebody actually
|
// visit. Nothing expires: a preview is a few kilobytes, and only files somebody actually
|
||||||
// scrolled past are ever made.
|
// scrolled past are ever made. The file is opened the first time something is stored, so a
|
||||||
type Cache struct{ Dir string }
|
// machine nobody browses with previews on never has one.
|
||||||
|
type Cache struct {
|
||||||
|
Path string
|
||||||
|
mu sync.Mutex
|
||||||
|
db *bolt.DB
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Entry is one cached preview, or the remembered reason there is none.
|
||||||
|
type Entry struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Engine string `json:"engine,omitempty"`
|
||||||
|
Codec string `json:"codec,omitempty"`
|
||||||
|
Width int `json:"width,omitempty"`
|
||||||
|
Height int `json:"height,omitempty"`
|
||||||
|
Made time.Time `json:"made"`
|
||||||
|
Failure string `json:"failure,omitempty"` // set instead of JPEG when ffmpeg refused the file
|
||||||
|
JPEG []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
bucketThumbs = []byte("thumbs")
|
||||||
|
bucketMeta = []byte("meta")
|
||||||
|
keyCount = []byte("count")
|
||||||
|
)
|
||||||
|
|
||||||
// Key identifies one file's preview. "v1" is the preview format: bump it to retire every
|
// Key identifies one file's preview. "v1" is the preview format: bump it to retire every
|
||||||
// cached preview at once.
|
// cached preview at once.
|
||||||
@@ -24,55 +55,164 @@ func Key(path string, size, mtime int64) string {
|
|||||||
return hex.EncodeToString(h[:])
|
return hex.EncodeToString(h[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c Cache) file(key, ext string) string { return filepath.Join(c.Dir, key[:2], key+ext) }
|
func NewCache(path string) *Cache { return &Cache{Path: path} }
|
||||||
|
|
||||||
// Get returns the preview, or the remembered reason it could not be made.
|
// open the database once; an open that failed is not retried, and previews still get made
|
||||||
func (c Cache) Get(key string) (jpeg []byte, failure string, found bool) {
|
// and served — they are simply not kept, and the log says why.
|
||||||
if len(key) < 2 {
|
func (c *Cache) open() (*bolt.DB, error) {
|
||||||
return nil, "", false
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.db != nil || c.err != nil {
|
||||||
|
return c.db, c.err
|
||||||
}
|
}
|
||||||
if b, err := os.ReadFile(c.file(key, ".jpg")); err == nil && len(b) > 0 {
|
db, err := bolt.Open(c.Path, 0o600, &bolt.Options{Timeout: 2 * time.Second})
|
||||||
return b, "", true
|
if err != nil {
|
||||||
|
c.err = err
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
if b, err := os.ReadFile(c.file(key, ".err")); err == nil {
|
if err := db.Update(func(tx *bolt.Tx) error {
|
||||||
return nil, string(b), true
|
if _, err := tx.CreateBucketIfNotExists(bucketThumbs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := tx.CreateBucketIfNotExists(bucketMeta)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
db.Close()
|
||||||
|
c.err = err
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return nil, "", false
|
c.db = db
|
||||||
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put keeps a preview; PutFailure keeps the reason there is none.
|
// Close releases the file. Previews after it are made but not kept.
|
||||||
func (c Cache) Put(key string, jpeg []byte) error {
|
func (c *Cache) Close() error {
|
||||||
if len(jpeg) == 0 {
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.db == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := c.db.Close()
|
||||||
|
c.db, c.err = nil, errors.New("cache closed")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns the entry for a key. A database that does not exist yet is a miss, not a reason
|
||||||
|
// to make one.
|
||||||
|
func (c *Cache) Get(key string) (Entry, bool, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
db := c.db
|
||||||
|
c.mu.Unlock()
|
||||||
|
if db == nil {
|
||||||
|
if _, err := os.Stat(c.Path); err != nil {
|
||||||
|
return Entry{}, false, nil
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if db, err = c.open(); err != nil {
|
||||||
|
return Entry{}, false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var e Entry
|
||||||
|
found := false
|
||||||
|
err := db.View(func(tx *bolt.Tx) error {
|
||||||
|
v := tx.Bucket(bucketThumbs).Get([]byte(key))
|
||||||
|
if v == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
e, err = decode(v) // copies: a value is only valid inside the transaction
|
||||||
|
found = err == nil
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
return e, found, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put stores an entry, replacing any under the same key.
|
||||||
|
func (c *Cache) Put(key string, e Entry) error {
|
||||||
|
if len(e.JPEG) == 0 && e.Failure == "" {
|
||||||
return errors.New("empty preview")
|
return errors.New("empty preview")
|
||||||
}
|
}
|
||||||
return c.write(c.file(key, ".jpg"), jpeg)
|
if e.Made.IsZero() {
|
||||||
}
|
e.Made = time.Now()
|
||||||
|
|
||||||
func (c Cache) PutFailure(key, reason string) error {
|
|
||||||
return c.write(c.file(key, ".err"), []byte(reason))
|
|
||||||
}
|
|
||||||
|
|
||||||
// write lands the file whole or not at all: a reader never sees a half-written JPEG.
|
|
||||||
func (c Cache) write(dst string, b []byte) error {
|
|
||||||
if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
tmp, err := os.CreateTemp(filepath.Dir(dst), ".tmp-*")
|
db, err := c.open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := tmp.Write(b); err != nil {
|
v, err := encode(e)
|
||||||
tmp.Close()
|
if err != nil {
|
||||||
os.Remove(tmp.Name())
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := tmp.Close(); err != nil {
|
return db.Update(func(tx *bolt.Tx) error {
|
||||||
os.Remove(tmp.Name())
|
b := tx.Bucket(bucketThumbs)
|
||||||
return err
|
k := []byte(key)
|
||||||
|
fresh := b.Get(k) == nil
|
||||||
|
if err := b.Put(k, v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if fresh {
|
||||||
|
m := tx.Bucket(bucketMeta)
|
||||||
|
return m.Put(keyCount, be64(count(m)+1))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats is how many previews are kept and how big the file is.
|
||||||
|
func (c *Cache) Stats() (entries int64, bytes int64) {
|
||||||
|
st, err := os.Stat(c.Path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0
|
||||||
}
|
}
|
||||||
if err := os.Rename(tmp.Name(), dst); err != nil {
|
bytes = st.Size()
|
||||||
os.Remove(tmp.Name())
|
db, err := c.open()
|
||||||
return err
|
if err != nil {
|
||||||
|
return 0, bytes
|
||||||
}
|
}
|
||||||
return nil
|
_ = db.View(func(tx *bolt.Tx) error { entries = count(tx.Bucket(bucketMeta)); return nil })
|
||||||
|
return entries, bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
func count(m *bolt.Bucket) int64 {
|
||||||
|
if v := m.Get(keyCount); len(v) == 8 {
|
||||||
|
return int64(binary.BigEndian.Uint64(v))
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func be64(n int64) []byte {
|
||||||
|
b := make([]byte, 8)
|
||||||
|
binary.BigEndian.PutUint64(b, uint64(n))
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// A value is a 4-byte header length, the header as JSON, then the JPEG bytes.
|
||||||
|
func encode(e Entry) ([]byte, error) {
|
||||||
|
hdr, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
v := make([]byte, 4+len(hdr)+len(e.JPEG))
|
||||||
|
binary.BigEndian.PutUint32(v, uint32(len(hdr)))
|
||||||
|
copy(v[4:], hdr)
|
||||||
|
copy(v[4+len(hdr):], e.JPEG)
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decode(v []byte) (Entry, error) {
|
||||||
|
if len(v) < 4 {
|
||||||
|
return Entry{}, errors.New("short value")
|
||||||
|
}
|
||||||
|
n := int(binary.BigEndian.Uint32(v))
|
||||||
|
if 4+n > len(v) {
|
||||||
|
return Entry{}, errors.New("bad header length")
|
||||||
|
}
|
||||||
|
var e Entry
|
||||||
|
if err := json.Unmarshal(v[4:4+n], &e); err != nil {
|
||||||
|
return Entry{}, err
|
||||||
|
}
|
||||||
|
if rest := v[4+n:]; len(rest) > 0 {
|
||||||
|
e.JPEG = append([]byte(nil), rest...)
|
||||||
|
}
|
||||||
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -107,30 +107,54 @@ func TestAttempts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCache(t *testing.T) {
|
func TestCache(t *testing.T) {
|
||||||
c := Cache{Dir: t.TempDir()}
|
c := NewCache(filepath.Join(t.TempDir(), "thumbs.db"))
|
||||||
k := Key("/a", 1, 2)
|
k := Key("/a", 1, 2)
|
||||||
if _, _, found := c.Get(k); found {
|
if _, found, err := c.Get(k); found || err != nil {
|
||||||
t.Fatal("empty cache found something")
|
t.Fatal("an empty cache found something, or complained")
|
||||||
}
|
}
|
||||||
if err := c.Put(k, []byte("jpeg")); err != nil {
|
if _, err := os.Stat(c.Path); err == nil {
|
||||||
|
t.Fatal("a miss must not create the database")
|
||||||
|
}
|
||||||
|
if n, b := c.Stats(); n != 0 || b != 0 {
|
||||||
|
t.Fatal("stats of nothing")
|
||||||
|
}
|
||||||
|
if err := c.Put(k, Entry{Path: "/a", Engine: "cpu", Codec: "png", Width: 640, Height: 360, JPEG: []byte("jpeg")}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if b, _, found := c.Get(k); !found || string(b) != "jpeg" {
|
e, found, err := c.Get(k)
|
||||||
t.Fatal("put then get")
|
if err != nil || !found || string(e.JPEG) != "jpeg" || e.Engine != "cpu" || e.Width != 640 || e.Path != "/a" || e.Made.IsZero() {
|
||||||
|
t.Fatalf("put then get: %+v %v %v", e, found, err)
|
||||||
}
|
}
|
||||||
k2 := Key("/b", 1, 2)
|
k2 := Key("/b", 1, 2)
|
||||||
if err := c.PutFailure(k2, "no video stream"); err != nil {
|
if err := c.Put(k2, Entry{Path: "/b", Failure: "no video stream"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if b, why, found := c.Get(k2); !found || b != nil || why != "no video stream" {
|
if e, found, _ := c.Get(k2); !found || e.JPEG != nil || e.Failure != "no video stream" {
|
||||||
t.Fatal("a failure is remembered with its reason")
|
t.Fatal("a failure is remembered with its reason")
|
||||||
}
|
}
|
||||||
if err := c.Put(k, nil); err == nil {
|
if err := c.Put(k, Entry{}); err == nil {
|
||||||
t.Fatal("an empty preview must not be kept")
|
t.Fatal("an empty preview must not be kept")
|
||||||
}
|
}
|
||||||
if left, _ := filepath.Glob(filepath.Join(c.Dir, "*", ".tmp-*")); len(left) != 0 {
|
// replacing an entry does not count it twice
|
||||||
t.Fatal("temp files left behind")
|
if err := c.Put(k, Entry{Path: "/a", JPEG: []byte("jpeg2")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
n, b := c.Stats()
|
||||||
|
if n != 2 || b == 0 {
|
||||||
|
t.Fatalf("stats = %d entries, %d bytes", n, b)
|
||||||
|
}
|
||||||
|
// survives a reopen
|
||||||
|
if err := c.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c2 := NewCache(c.Path)
|
||||||
|
if e, found, _ := c2.Get(k); !found || string(e.JPEG) != "jpeg2" {
|
||||||
|
t.Fatal("not there after reopening")
|
||||||
|
}
|
||||||
|
if n, _ := c2.Stats(); n != 2 {
|
||||||
|
t.Fatal("count lost on reopen")
|
||||||
|
}
|
||||||
|
c2.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// The real thing, when ffmpeg is here: a PNG and a generated video each come back as a
|
// The real thing, when ffmpeg is here: a PNG and a generated video each come back as a
|
||||||
|
|||||||
+2
-2
@@ -192,7 +192,7 @@ export default function Files({ onOpenInTerminal }: { onOpenInTerminal?: (cmd: s
|
|||||||
const [thumbs, setThumbs] = useState<ThumbStatus | null>(null)
|
const [thumbs, setThumbs] = useState<ThumbStatus | null>(null)
|
||||||
const [listEl, setListEl] = useState<HTMLDivElement | null>(null)
|
const [listEl, setListEl] = useState<HTMLDivElement | null>(null)
|
||||||
const near = useMemo(() => (listEl && previews ? new Near(listEl) : null), [listEl, previews])
|
const near = useMemo(() => (listEl && previews ? new Near(listEl) : null), [listEl, previews])
|
||||||
const readThumbs = useCallback((refresh = false) => { api.thumbStatus(sudo, refresh).then(setThumbs).catch((e) => setThumbs({ ok: false, engine: '', summary: '', gpus: [], error: e instanceof Error ? e.message : 'failed', user: '', cache_dir: '' })) }, [sudo])
|
const readThumbs = useCallback((refresh = false) => { api.thumbStatus(sudo, refresh).then(setThumbs).catch((e) => setThumbs({ ok: false, engine: '', summary: '', gpus: [], error: e instanceof Error ? e.message : 'failed', user: '', cache: { path: '', entries: 0, bytes: 0 } })) }, [sudo])
|
||||||
useEffect(() => { try { localStorage.setItem(PREVIEWS_KEY, previews ? '1' : '0') } catch { /* private mode */ } }, [previews])
|
useEffect(() => { try { localStorage.setItem(PREVIEWS_KEY, previews ? '1' : '0') } catch { /* private mode */ } }, [previews])
|
||||||
useEffect(() => { if (previews) readThumbs(); else setThumbs(null) }, [previews, readThumbs])
|
useEffect(() => { if (previews) readThumbs(); else setThumbs(null) }, [previews, readThumbs])
|
||||||
|
|
||||||
@@ -279,7 +279,7 @@ export default function Files({ onOpenInTerminal }: { onOpenInTerminal?: (cmd: s
|
|||||||
<div className="files-tools">
|
<div className="files-tools">
|
||||||
<button className={'btn' + (sudo ? ' on' : '')} title="browse as root" onClick={() => setSudo(!sudo)}>{sudo ? 'root' : 'sudo'}</button>
|
<button className={'btn' + (sudo ? ' on' : '')} title="browse as root" onClick={() => setSudo(!sudo)}>{sudo ? 'root' : 'sudo'}</button>
|
||||||
<button className={'btn' + (hidden ? ' on' : '')} title="show hidden files" onClick={() => setHidden(!hidden)}>.hidden</button>
|
<button className={'btn' + (hidden ? ' on' : '')} title="show hidden files" onClick={() => setHidden(!hidden)}>.hidden</button>
|
||||||
<button className={'btn' + (previews ? ' on' : '')} title={'thumbnails for images and videos' + (thumbs ? ` — ${thumbs.summary}` : '')} onClick={() => setPreviews(!previews)}>previews</button>
|
<button className={'btn' + (previews ? ' on' : '')} title={'thumbnails for images and videos' + (thumbs ? ` — ${thumbs.summary}` + (thumbs.cache.entries ? ` · ${thumbs.cache.entries} kept (${fmtBytes(thumbs.cache.bytes)})` : '') : '')} onClick={() => setPreviews(!previews)}>previews</button>
|
||||||
<select className="btn" value={sort} onChange={(e) => setSort(e.target.value as Sort)}><option value="name">name</option><option value="size">size</option><option value="mtime">modified</option></select>
|
<select className="btn" value={sort} onChange={(e) => setSort(e.target.value as Sort)}><option value="name">name</option><option value="size">size</option><option value="mtime">modified</option></select>
|
||||||
<input className="filter" placeholder="filter" value={q} onChange={(e) => setQ(e.target.value)} />
|
<input className="filter" placeholder="filter" value={q} onChange={(e) => setQ(e.target.value)} />
|
||||||
<button className="btn" onClick={mkdir}>+ folder</button>
|
<button className="btn" onClick={mkdir}>+ folder</button>
|
||||||
|
|||||||
+1
-1
@@ -252,7 +252,7 @@ export interface ARCView { detail: ARCDetail; live: ARCLive | null }
|
|||||||
export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean; preview?: 'image' | 'video' }
|
export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean; preview?: 'image' | 'video' }
|
||||||
export interface ThumbGPU { api: string; device: string; name?: string; codecs?: string[]; reason?: string; permission?: boolean }
|
export interface ThumbGPU { api: string; device: string; name?: string; codecs?: string[]; reason?: string; permission?: boolean }
|
||||||
// How previews get made on the machine: ok with an engine, or why not and what would fix it.
|
// How previews get made on the machine: ok with an engine, or why not and what would fix it.
|
||||||
export interface ThumbStatus { ok: boolean; engine: 'gpu' | 'cpu' | ''; summary: string; ffmpeg?: string; gpus: ThumbGPU[]; unusable?: ThumbGPU[]; error?: string; install?: { pm: string; package: string }; gpu_fix?: boolean; user: string; cache_dir: string }
|
export interface ThumbStatus { ok: boolean; engine: 'gpu' | 'cpu' | ''; summary: string; ffmpeg?: string; gpus: ThumbGPU[]; unusable?: ThumbGPU[]; error?: string; install?: { pm: string; package: string }; gpu_fix?: boolean; user: string; cache: { path: string; entries: number; bytes: number } }
|
||||||
export interface FileInfo extends FileEntry { path: string; text: boolean }
|
export interface FileInfo extends FileEntry { path: string; text: boolean }
|
||||||
|
|
||||||
const q = (path: string, sudo: boolean, extra = '') => `path=${encodeURIComponent(path)}${sudo ? '&sudo=1' : ''}${extra}`
|
const q = (path: string, sudo: boolean, extra = '') => `path=${encodeURIComponent(path)}${sudo ? '&sudo=1' : ''}${extra}`
|
||||||
|
|||||||
Reference in New Issue
Block a user