package media // What to run when a video is played in the browser: the qualities it is offered in (the // ladder), the ways a machine could make one of them on the device asked for, in the order // worth trying, and the ffmpeg argument list for one piece-making run. All decisions, no // processes, so it can be tested. Everything is H.264 High and AAC: what every browser plays. import ( "fmt" "math" "sort" "strconv" "strings" ) // Rung is one quality the player can pick: a size, a frame rate and a bit rate. type Rung struct { Name string `json:"name"` // in URLs: the width, "1920" Label string `json:"label"` // what the menu says: "1080s", "720p" Width int `json:"width"` Height int `json:"height"` FPS float64 `json:"fps"` Bitrate int `json:"bitrate"` // video, bits a second, on average Peak int `json:"peak"` // what the playlist promises, sound included level string // H.264 level, "5.2" codecs string // for the master playlist } // One step of the ladder: the long side for a 360 video (its width), the short side for a flat // one, and the bit rate (Mb/s ×10) at more than 30 frames a second and at 30 or fewer. type step struct { size int label string hi, lo int cap30 bool } // A 360 picture is seen a quarter of its width at a time, so it needs more pixels than a flat // one for the same sharpness; YouTube names its sizes by the flat picture they look like // ("1080s"), and so does this. var vrSteps = []step{ {3840, "2160s", 220, 160, false}, {2880, "1440s", 120, 90, false}, {1920, "1080s", 60, 45, false}, {1280, "720s", 25, 25, true}, } var flatSteps = []step{ {2160, "2160p", 160, 120, false}, {1440, "1440p", 90, 65, false}, {1080, "1080p", 50, 35, false}, {720, "720p", 25, 25, true}, {480, "480p", 12, 12, true}, } const audioRate = 160_000 func even(x float64) int { return int(math.Round(x/2)) * 2 } // Ladder is the qualities a video is offered in, best first: every step no bigger than the // picture itself, or the picture's own size if it is smaller than all of them. func Ladder(i Info) []Rung { v := i.Video if v == nil || v.Width <= 0 || v.Height <= 0 { return nil } vr := v.Spherical steps := flatSteps w, h := v.Shown() src := min(w, h) // a flat picture is named by its short side if vr { steps, src = vrSteps, w } // A picture between two steps (a 2292×960 capture) gets its own size on top, at the bit // rate of the step above, so the best on offer is never less than the file. var out []Rung for k, s := range steps { if s.size > src { continue } if len(out) == 0 && s.size < src && k > 0 { // never above the top step: H.264 stops there own := steps[k-1] own.size, own.label, own.cap30 = src, strconv.Itoa(src)+map[bool]string{true: "s", false: "p"}[vr], false out = append(out, rung(i, own)) } out = append(out, rung(i, s)) } if len(out) == 0 { last := steps[len(steps)-1] last.size, last.label = src, strconv.Itoa(src)+map[bool]string{true: "s", false: "p"}[vr] out = append(out, rung(i, last)) } return out } func rung(i Info, s step) Rung { v := i.Video sw, sh := v.Shown() var w, h int switch { case v.Spherical: w, h = s.size, even(float64(s.size)*float64(sh)/float64(sw)) case sw >= sh: h, w = s.size, even(float64(s.size)*float64(sw)/float64(sh)) default: // standing up w, h = s.size, even(float64(s.size)*float64(sh)/float64(sw)) } fps := v.FPS if fps <= 0 { fps = 30 } for fps > 60.5 || (s.cap30 && fps > 30.5) { fps /= 2 } fps = math.Round(fps*1000) / 1000 br := s.lo if fps > 30.5 { br = s.hi } r := Rung{Name: strconv.Itoa(w), Label: s.label, Width: w, Height: h, FPS: fps, Bitrate: br * 100_000} r.Peak = r.Bitrate*3/2 + audioRate r.level = level(w, h, fps) lv, _ := strconv.ParseFloat(r.level, 64) r.codecs = fmt.Sprintf("avc1.6400%02x", int(math.Round(lv*10))) if i.Audio != nil { r.codecs += ",mp4a.40.2" } return r } // level is the lowest H.264 level that takes this many macroblocks a frame and a second. func level(w, h int, fps float64) string { mbs := ((w + 15) / 16) * ((h + 15) / 16) rate := float64(mbs) * fps for _, l := range []struct { name string mbps float64 frame int }{ {"3.1", 108000, 3600}, {"3.2", 216000, 5120}, {"4.0", 245760, 8192}, {"4.2", 522240, 8704}, {"5.0", 589824, 22080}, {"5.1", 983040, 36864}, {"5.2", 2073600, 36864}, } { if rate <= l.mbps && mbs <= l.frame { return l.name } } return "6.2" } // fpsArg is a frame rate as ffmpeg's fps filter takes it, exactly for the NTSC rates. func fpsArg(f float64) string { for _, n := range []float64{24, 30, 60, 120} { if math.Abs(f-n*1000/1001) < 0.01 { return fmt.Sprintf("%d/1001", int(n*1000)) } } return strings.TrimRight(strings.TrimRight(strconv.FormatFloat(f, 'f', 3, 64), "0"), ".") } // Recipe is one way to make a quality: the engine's name for the record, the device it runs // on ("cpu" for the software encoder), and the ffmpeg arguments that go before the input, the // video filter, and the video codec. type Recipe struct { Engine string Device string Pre []string VF string Video []string } // CPU is the device name for software encoding. const CPU = "cpu" // Device is one thing a machine can make H.264 on, for the Transcode device setting. type Device struct { ID string `json:"id"` // a render node, or "cpu" Name string `json:"name"` // "Intel Arc A380", "CPU (libx264)" Integrated bool `json:"integrated,omitempty"` OK bool `json:"ok"` Reason string `json:"reason,omitempty"` // when not OK: why } // Devices lists what this machine could make the player's H.264 on: every GPU, usable or not // (and why not), then the CPU. func Devices(c Caps) []Device { var out []Device if c.Error != "" { return []Device{{ID: CPU, Name: "CPU", Reason: c.Error}} } for _, g := range c.GPUs { d := Device{ID: g.Device, Name: g.Name, Integrated: g.Integrated} if d.Name == "" { d.Name = g.Device } switch { case g.API != "vaapi": d.Reason = "only VAAPI devices encode here" case g.Encodes != nil && !has(g.Encodes, "h264"): d.Reason = "doesn't encode H.264" default: d.OK = true } out = append(out, d) } for _, g := range c.Unusable { name := g.Name if name == "" { name = g.Device } out = append(out, Device{ID: g.Device, Name: name, Integrated: g.Integrated, Reason: g.Reason}) } d := Device{ID: CPU, Name: "CPU (libx264)", OK: has(c.Encoders, "libx264")} if !d.OK { d.Reason = "this ffmpeg has no libx264" } return append(out, d) } // recipes are the ways to make quality r of src on device, best first: on a GPU, decoding on // the same card (frames never leave it) before decoding in software and uploading; the CPU // last, as the fallback whatever was asked. device "" is any: a card on the bus before the // integrated GPU. An empty list means nothing here can. func recipes(c Caps, device string, src Info, r Rung) []Recipe { var out []Recipe gpus := append([]GPU(nil), c.GPUs...) sort.SliceStable(gpus, func(i, j int) bool { return !gpus[i].Integrated && gpus[j].Integrated }) kbps := r.Bitrate / 1000 rate := []string{"-b:v", strconv.Itoa(kbps) + "k", "-maxrate", strconv.Itoa(kbps*3/2) + "k", "-bufsize", strconv.Itoa(kbps*2) + "k", "-profile:v", "high", "-level", r.level} fps := "" if src.Video == nil || src.Video.FPS <= 0 || math.Abs(src.Video.FPS-r.FPS) > 0.01 { fps = "fps=" + fpsArg(r.FPS) + "," } if device != CPU { for i := range gpus { g := &gpus[i] // Encodes is vainfo's answer; without vainfo there is none, and the card is tried // anyway, with the next recipe behind it. if g.API != "vaapi" || (g.Encodes != nil && !has(g.Encodes, "h264")) || (device != "" && g.Device != device) { continue } video := append(append([]string{"-c:v", "h264_vaapi"}, rate...), "-bf", "0") name := g.Name if name == "" { name = g.Device } // The card doesn't turn a phone's picture by itself (ffmpeg's own turning is for // pictures in memory), so it is told to, and ffmpeg is told not to. scale := turn(src) + fmt.Sprintf("scale_vaapi=w=%d:h=%d:format=nv12", r.Width, r.Height) // HDR10 (PQ) is tone-mapped on the card, which needs the file's mastering data; // HLG is made to look right on an ordinary screen as it is, so it is only scaled // (tonemap_vaapi refuses it). Either way the next recipe tone-maps in software. if src.Video != nil && has(g.Codecs, src.Video.Codec) { vf := fps + scale if src.PQ() { vf = fps + "tonemap_vaapi=format=nv12:t=bt709:p=bt709:m=bt709," + scale } out = append(out, Recipe{Engine: name, Device: g.Device, Pre: []string{"-noautorotate", "-hwaccel", "vaapi", "-hwaccel_device", g.Device, "-hwaccel_output_format", "vaapi"}, VF: vf, Video: video}) } cpuVF := fmt.Sprintf("%sformat=nv12,scale=%d:%d,hwupload", fps, r.Width, r.Height) if src.PQ() { cpuVF = fps + toneMap + fmt.Sprintf(",scale=%d:%d,format=nv12,hwupload", r.Width, r.Height) } out = append(out, Recipe{Engine: name + ", decoded on the CPU", Device: g.Device, Pre: []string{"-init_hw_device", "vaapi=va:" + g.Device, "-filter_hw_device", "va"}, VF: cpuVF, Video: video}) } } if has(c.Encoders, "libx264") { vf := fmt.Sprintf("%sscale=%d:%d,format=yuv420p", fps, r.Width, r.Height) if src.PQ() { vf = fps + toneMap + fmt.Sprintf(",scale=%d:%d,format=yuv420p", r.Width, r.Height) } out = append(out, Recipe{Engine: "CPU (libx264)", Device: CPU, VF: vf, Video: append([]string{"-c:v", "libx264", "-preset", "veryfast", "-sc_threshold", "0"}, rate...)}) } return out } // turn is the card's filter that turns a picture the way it is meant to be shown. func turn(src Info) string { if src.Video == nil { return "" } switch src.Video.Rotation { case 90: return "transpose_vaapi=dir=clock," case 180: return "transpose_vaapi=dir=reversal," case 270: return "transpose_vaapi=dir=cclock," } return "" } // toneMap turns an HDR10 picture into an ordinary one, in software. const toneMap = "zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable,zscale=t=bt709:m=bt709:r=tv" // Seg is the length of a piece, in seconds. const Seg = 3.0 // tsBase is added to every run's times, so pieces from different runs follow on: without it // a run from the top would start just below zero (the B-frame and AAC delay) and the muxer // would shift that run alone. const tsBase = 10 // pieceArgs is one run making quality r from piece n on, into dir as .ts: an accurate seek // to n×Seg, a keyframe on every Seg mark and nowhere else (so every piece starts on one, and // pieces from any run and quality line up), times put back where that is in the film. in is a // path or, for a file on another machine, an http URL. func pieceArgs(dir, in string, src Info, rc Recipe, r Rung, n int) []string { at := strconv.FormatFloat(float64(n)*Seg, 'f', 3, 64) a := []string{"-nostdin", "-hide_banner", "-loglevel", "error", "-y"} a = append(a, rc.Pre...) if strings.HasPrefix(in, "http://") || strings.HasPrefix(in, "https://") { a = append(a, "-reconnect", "1", "-reconnect_on_network_error", "1", "-reconnect_delay_max", "4") } a = append(a, "-ss", at, "-i", in, "-map", "0:v:0", "-map", "0:a:0?", "-sn", "-dn") if rc.VF != "" { a = append(a, "-vf", rc.VF) } a = append(a, rc.Video...) if src.Audio == nil { a = append(a, "-an") } else { a = append(a, "-c:a", "aac", "-b:a", "160k", "-ac", "2") } gop := int(math.Ceil(r.FPS*Seg)) * 4 a = append(a, "-g", strconv.Itoa(gop), "-force_key_frames", fmt.Sprintf("expr:gte(t,n_forced*%g)", Seg), "-output_ts_offset", strconv.FormatFloat(float64(n)*Seg+tsBase, 'f', 3, 64), "-f", "hls", "-hls_time", strconv.FormatFloat(Seg-0.5, 'f', 2, 64), "-hls_list_size", "0", "-hls_flags", "temp_file", "-hls_segment_type", "mpegts", "-start_number", strconv.Itoa(n), "-hls_segment_filename", dir+"/%d.ts", dir+"/ffmpeg.m3u8") return a } func has(list []string, x string) bool { for _, v := range list { if v == x { return true } } return false }