// The 360° view: an equirectangular video drawn onto a canvas as the flat picture a person // would see looking out from the middle of it. Drag (or swipe) to look around, pinch or scroll // to zoom, and on a phone, optionally, turn the phone. One full-canvas triangle pair; the // fragment shader works out, for each pixel, which way it looks and where that is in the video. const VERT = ` attribute vec2 a; varying vec2 p; void main() { p = a; gl_Position = vec4(a, 0.0, 1.0); }` const FRAG = ` precision highp float; uniform sampler2D tex; uniform mat3 view; // columns: right, up, forward uniform vec2 aspect; // the screen's sides over its taller side: (w/h, 1) or (1, h/w) uniform float fov; // radians, across the taller side varying vec2 p; const float PI = 3.14159265358979; void main() { vec2 q = p * aspect; // Narrow: an ordinary lens. Wide: stereographic, which goes on past 180 degrees to the // "little planet". Between 100 and 140 degrees one turns into the other. vec3 rect = normalize(vec3(q * tan(min(fov, 3.0) * 0.5), 1.0)); float r = length(q) * 2.0 * tan(fov * 0.25); float th = 2.0 * atan(r * 0.5); vec2 dir = r > 0.0 ? q / length(q) : vec2(0.0); vec3 stereo = vec3(dir * sin(th), cos(th)); float t = smoothstep(1.745, 2.443, fov); vec3 d = normalize(view * normalize(mix(rect, stereo, t))); float lon = atan(d.x, -d.z); float lat = asin(clamp(d.y, -1.0, 1.0)); gl_FragColor = texture2D(tex, vec2(lon / (2.0 * PI) + 0.5, 0.5 - lat / PI)); }` // From a close look at something far off to the whole sphere as a little planet. const MIN_FOV = 8, MAX_FOV = 300 type Vec3 = [number, number, number] type Quat = [number, number, number, number] // x, y, z, w function qmul(a: Quat, b: Quat): Quat { const [ax, ay, az, aw] = a, [bx, by, bz, bw] = b return [aw * bx + ax * bw + ay * bz - az * by, aw * by - ax * bz + ay * bw + az * bx, aw * bz + ax * by - ay * bx + az * bw, aw * bw - ax * bx - ay * by - az * bz] } function qrot(q: Quat, v: Vec3): Vec3 { const [x, y, z, w] = q const ix = w * v[0] + y * v[2] - z * v[1], iy = w * v[1] + z * v[0] - x * v[2], iz = w * v[2] + x * v[1] - y * v[0], iw = -x * v[0] - y * v[1] - z * v[2] return [ix * w + iw * -x + iy * -z - iz * -y, iy * w + iw * -y + iz * -x - ix * -z, iz * w + iw * -z + ix * -y - iy * -x] } const axis = (v: Vec3, rad: number): Quat => { const s = Math.sin(rad / 2); return [v[0] * s, v[1] * s, v[2] * s, Math.cos(rad / 2)] } // The phone's orientation as a camera's, the way the W3C spec defines the angles: Z-X'-Y'' // intrinsic, then turned to look out of the back of the phone, then for the screen's rotation. function deviceQuat(alpha: number, beta: number, gamma: number, screen: number): Quat { const d = Math.PI / 180 const [a, b, g] = [alpha * d, beta * d, gamma * d] let q = qmul(qmul(axis([0, 1, 0], a), axis([1, 0, 0], b)), axis([0, 0, 1], -g)) // YXZ, in three.js's frame q = qmul(q, [-Math.SQRT1_2, 0, 0, Math.SQRT1_2]) return qmul(q, axis([0, 0, 1], -screen * d)) } export class Sphere { yaw = 0 // degrees, + is right pitch = 0 // degrees, + is up fov = 80 // degrees, the taller of the two sides private gl: WebGLRenderingContext private tex: WebGLTexture private uView: WebGLUniformLocation private uAspect: WebGLUniformLocation private uFov: WebGLUniformLocation private raf = 0 private frameCb = 0 private dirty = true private hasFrame = false private gyro: { alpha: number; beta: number; gamma: number } | null = null private gyroYaw = 0 // what dragging adds to the phone's own heading private ro: ResizeObserver maxTexture: number constructor(private canvas: HTMLCanvasElement, private video: HTMLVideoElement) { const gl = canvas.getContext('webgl', { antialias: false, alpha: false, preserveDrawingBuffer: false }) if (!gl) throw new Error('This browser can’t draw 360° video (no WebGL).') this.gl = gl this.maxTexture = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number const sh = (type: number, src: string) => { const s = gl.createShader(type)! gl.shaderSource(s, src); gl.compileShader(s) if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s) ?? 'shader') return s } const prog = gl.createProgram()! gl.attachShader(prog, sh(gl.VERTEX_SHADER, VERT)); gl.attachShader(prog, sh(gl.FRAGMENT_SHADER, FRAG)) gl.linkProgram(prog); gl.useProgram(prog) const buf = gl.createBuffer() gl.bindBuffer(gl.ARRAY_BUFFER, buf) gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW) const loc = gl.getAttribLocation(prog, 'a') gl.enableVertexAttribArray(loc); gl.vertexAttribPointer(loc, 2, gl.FLOAT, false, 0, 0) this.tex = gl.createTexture()! gl.bindTexture(gl.TEXTURE_2D, this.tex) // No mipmaps (a video changes every frame) and clamped: any size of picture will do. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR) gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR) gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 0])) this.uView = gl.getUniformLocation(prog, 'view')! this.uAspect = gl.getUniformLocation(prog, 'aspect')! this.uFov = gl.getUniformLocation(prog, 'fov')! this.ro = new ResizeObserver(() => { this.dirty = true }) this.ro.observe(canvas) this.watchFrames() const loop = () => { this.draw(); this.raf = requestAnimationFrame(loop) } this.raf = requestAnimationFrame(loop) } // A new picture from the video: on the frame callback where there is one, else every draw // while it plays. private watchFrames() { const v = this.video as HTMLVideoElement & { requestVideoFrameCallback?: (cb: () => void) => number } if (v.requestVideoFrameCallback) { const cb = () => { this.upload(); this.frameCb = v.requestVideoFrameCallback!(cb) } this.frameCb = v.requestVideoFrameCallback(cb) } v.addEventListener('loadeddata', this.upload) v.addEventListener('seeked', this.upload) } private upload = () => { const v = this.video if (v.readyState < 2 || !v.videoWidth) return const gl = this.gl gl.bindTexture(gl.TEXTURE_2D, this.tex) try { gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, v) this.hasFrame = true this.dirty = true } catch { /* not ready yet */ } } private draw() { const v = this.video as HTMLVideoElement & { requestVideoFrameCallback?: unknown } if (!v.requestVideoFrameCallback && !v.paused) this.upload() if (!this.dirty && !this.gyro) return this.dirty = false const c = this.canvas, dpr = Math.min(window.devicePixelRatio || 1, 2) const w = Math.round(c.clientWidth * dpr), h = Math.round(c.clientHeight * dpr) if (!w || !h) return if (c.width !== w || c.height !== h) { c.width = w; c.height = h } const gl = this.gl gl.viewport(0, 0, w, h) if (!this.hasFrame) { gl.clearColor(0, 0, 0, 1); gl.clear(gl.COLOR_BUFFER_BIT); return } // The field of view applies to the taller side; on a phone held upright that is the height. let right: Vec3, up: Vec3, fwd: Vec3 if (this.gyro) { const q = qmul(axis([0, 1, 0], (-this.gyroYaw * Math.PI) / 180), deviceQuat(this.gyro.alpha, this.gyro.beta, this.gyro.gamma, screenAngle())) right = qrot(q, [1, 0, 0]); up = qrot(q, [0, 1, 0]); fwd = qrot(q, [0, 0, -1]) } else { const y = (this.yaw * Math.PI) / 180, p = (this.pitch * Math.PI) / 180 fwd = [Math.sin(y) * Math.cos(p), Math.sin(p), -Math.cos(y) * Math.cos(p)] right = [Math.cos(y), 0, Math.sin(y)] up = [right[1] * fwd[2] - right[2] * fwd[1], right[2] * fwd[0] - right[0] * fwd[2], right[0] * fwd[1] - right[1] * fwd[0]] } // d = right·x + up·y + fwd·1, column-major gl.uniformMatrix3fv(this.uView, false, new Float32Array([...right, ...up, ...fwd])) gl.uniform2f(this.uAspect, w >= h ? 1 : w / h, w >= h ? h / w : 1) gl.uniform1f(this.uFov, (this.fov * Math.PI) / 180) gl.drawArrays(gl.TRIANGLES, 0, 6) } // Dragging by a pixel turns the view by the angle a pixel covers, so what is under the finger // stays under it. drag(dx: number, dy: number) { const per = this.fov / Math.max(this.canvas.clientWidth, this.canvas.clientHeight) if (this.gyro) this.gyroYaw = (this.gyroYaw - dx * per) % 360 else { this.yaw = (this.yaw - dx * per) % 360 this.pitch = Math.max(-89, Math.min(89, this.pitch + dy * per)) } this.dirty = true } zoom(factor: number) { this.fov = Math.max(MIN_FOV, Math.min(MAX_FOV, this.fov * factor)) this.dirty = true } get gyroOn() { return this.gyro !== null } private onOrient = (e: DeviceOrientationEvent) => { if (e.alpha === null || e.beta === null || e.gamma === null) return if (!this.gyro) { // Keep looking where the drag left off: the phone's heading now is the drag's yaw. const q = deviceQuat(e.alpha, e.beta, e.gamma, screenAngle()) const f = qrot(q, [0, 0, -1]) this.gyroYaw = this.yaw - (Math.atan2(f[0], -f[2]) * 180) / Math.PI } this.gyro = { alpha: e.alpha, beta: e.beta, gamma: e.gamma } } // Turning on motion asks for permission on an iPhone, which must happen in a tap. async setGyro(on: boolean): Promise { if (!on) { window.removeEventListener('deviceorientation', this.onOrient) if (this.gyro) { const q = qmul(axis([0, 1, 0], (-this.gyroYaw * Math.PI) / 180), deviceQuat(this.gyro.alpha, this.gyro.beta, this.gyro.gamma, screenAngle())) const f = qrot(q, [0, 0, -1]) this.yaw = (Math.atan2(f[0], -f[2]) * 180) / Math.PI this.pitch = (Math.asin(Math.max(-1, Math.min(1, f[1]))) * 180) / Math.PI } this.gyro = null this.dirty = true return false } const D = DeviceOrientationEvent as unknown as { requestPermission?: () => Promise } if (D.requestPermission) { try { if ((await D.requestPermission()) !== 'granted') return false } catch { return false } } window.addEventListener('deviceorientation', this.onOrient) return true } destroy() { cancelAnimationFrame(this.raf) const v = this.video as HTMLVideoElement & { cancelVideoFrameCallback?: (n: number) => void } if (this.frameCb && v.cancelVideoFrameCallback) v.cancelVideoFrameCallback(this.frameCb) v.removeEventListener('loadeddata', this.upload) v.removeEventListener('seeked', this.upload) window.removeEventListener('deviceorientation', this.onOrient) this.ro.disconnect() this.gl.getExtension('WEBGL_lose_context')?.loseContext() } } function screenAngle(): number { const o = screen.orientation?.angle if (typeof o === 'number') return o return (window as unknown as { orientation?: number }).orientation ?? 0 } export const canMotion = () => typeof DeviceOrientationEvent !== 'undefined' && matchMedia('(pointer: coarse)').matches