hamlab/web/src/vr.ts

// The 360° view: an equirectangular video drawn onto a canvas as the flat picture a person
// would see looking out from the middle of it. Drag (or swipe) to look around, pinch or scroll
// to zoom, and on a phone, optionally, turn the phone. One full-canvas triangle pair; the
// fragment shader works out, for each pixel, which way it looks and where that is in the video.

const VERT = `
attribute vec2 a;
varying vec2 p;
void main() { p = a; gl_Position = vec4(a, 0.0, 1.0); }`

const FRAG = `
precision highp float;
uniform sampler2D tex;
uniform mat3 view;  // columns: right, up, forward
uniform vec2 aspect; // the screen's sides over its taller side: (w/h, 1) or (1, h/w)
uniform float fov;   // radians, across the taller side
varying vec2 p;
const float PI = 3.14159265358979;
void main() {
  vec2 q = p * aspect;
  // Narrow: an ordinary lens. Wide: stereographic, which goes on past 180 degrees to the
  // "little planet". Between 100 and 140 degrees one turns into the other.
  vec3 rect = normalize(vec3(q * tan(min(fov, 3.0) * 0.5), 1.0));
  float r = length(q) * 2.0 * tan(fov * 0.25);
  float th = 2.0 * atan(r * 0.5);
  vec2 dir = r > 0.0 ? q / length(q) : vec2(0.0);
  vec3 stereo = vec3(dir * sin(th), cos(th));
  float t = smoothstep(1.745, 2.443, fov);
  vec3 d = normalize(view * normalize(mix(rect, stereo, t)));
  float lon = atan(d.x, -d.z);
  float lat = asin(clamp(d.y, -1.0, 1.0));
  gl_FragColor = texture2D(tex, vec2(lon / (2.0 * PI) + 0.5, 0.5 - lat / PI));
}`

// From a close look at something far off to the whole sphere as a little planet.
const MIN_FOV = 8, MAX_FOV = 300

type Vec3 = [number, number, number]
type Quat = [number, number, number, number] // x, y, z, w

function qmul(a: Quat, b: Quat): Quat {
  const [ax, ay, az, aw] = a, [bx, by, bz, bw] = b
  return [aw * bx + ax * bw + ay * bz - az * by, aw * by - ax * bz + ay * bw + az * bx, aw * bz + ax * by - ay * bx + az * bw, aw * bw - ax * bx - ay * by - az * bz]
}
function qrot(q: Quat, v: Vec3): Vec3 {
  const [x, y, z, w] = q
  const ix = w * v[0] + y * v[2] - z * v[1], iy = w * v[1] + z * v[0] - x * v[2], iz = w * v[2] + x * v[1] - y * v[0], iw = -x * v[0] - y * v[1] - z * v[2]
  return [ix * w + iw * -x + iy * -z - iz * -y, iy * w + iw * -y + iz * -x - ix * -z, iz * w + iw * -z + ix * -y - iy * -x]
}
const axis = (v: Vec3, rad: number): Quat => { const s = Math.sin(rad / 2); return [v[0] * s, v[1] * s, v[2] * s, Math.cos(rad / 2)] }

// The phone's orientation as a camera's, the way the W3C spec defines the angles: Z-X'-Y''
// intrinsic, then turned to look out of the back of the phone, then for the screen's rotation.
function deviceQuat(alpha: number, beta: number, gamma: number, screen: number): Quat {
  const d = Math.PI / 180
  const [a, b, g] = [alpha * d, beta * d, gamma * d]
  let q = qmul(qmul(axis([0, 1, 0], a), axis([1, 0, 0], b)), axis([0, 0, 1], -g)) // YXZ, in three.js's frame
  q = qmul(q, [-Math.SQRT1_2, 0, 0, Math.SQRT1_2])
  return qmul(q, axis([0, 0, 1], -screen * d))
}

export class Sphere {
  yaw = 0 // degrees, + is right
  pitch = 0 // degrees, + is up
  fov = 80 // degrees, the taller of the two sides
  private gl: WebGLRenderingContext
  private tex: WebGLTexture
  private uView: WebGLUniformLocation
  private uAspect: WebGLUniformLocation
  private uFov: WebGLUniformLocation
  private raf = 0
  private frameCb = 0
  private dirty = true
  private hasFrame = false
  private gyro: { alpha: number; beta: number; gamma: number } | null = null
  private gyroYaw = 0 // what dragging adds to the phone's own heading
  private ro: ResizeObserver
  maxTexture: number

  constructor(private canvas: HTMLCanvasElement, private video: HTMLVideoElement) {
    const gl = canvas.getContext('webgl', { antialias: false, alpha: false, preserveDrawingBuffer: false })
    if (!gl) throw new Error('This browser can’t draw 360° video (no WebGL).')
    this.gl = gl
    this.maxTexture = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number
    const sh = (type: number, src: string) => {
      const s = gl.createShader(type)!
      gl.shaderSource(s, src); gl.compileShader(s)
      if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s) ?? 'shader')
      return s
    }
    const prog = gl.createProgram()!
    gl.attachShader(prog, sh(gl.VERTEX_SHADER, VERT)); gl.attachShader(prog, sh(gl.FRAGMENT_SHADER, FRAG))
    gl.linkProgram(prog); gl.useProgram(prog)
    const buf = gl.createBuffer()
    gl.bindBuffer(gl.ARRAY_BUFFER, buf)
    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW)
    const loc = gl.getAttribLocation(prog, 'a')
    gl.enableVertexAttribArray(loc); gl.vertexAttribPointer(loc, 2, gl.FLOAT, false, 0, 0)
    this.tex = gl.createTexture()!
    gl.bindTexture(gl.TEXTURE_2D, this.tex)
    // No mipmaps (a video changes every frame) and clamped: any size of picture will do.
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 0]))
    this.uView = gl.getUniformLocation(prog, 'view')!
    this.uAspect = gl.getUniformLocation(prog, 'aspect')!
    this.uFov = gl.getUniformLocation(prog, 'fov')!
    this.ro = new ResizeObserver(() => { this.dirty = true })
    this.ro.observe(canvas)
    this.watchFrames()
    const loop = () => { this.draw(); this.raf = requestAnimationFrame(loop) }
    this.raf = requestAnimationFrame(loop)
  }

  // A new picture from the video: on the frame callback where there is one, else every draw
  // while it plays.
  private watchFrames() {
    const v = this.video as HTMLVideoElement & { requestVideoFrameCallback?: (cb: () => void) => number }
    if (v.requestVideoFrameCallback) {
      const cb = () => { this.upload(); this.frameCb = v.requestVideoFrameCallback!(cb) }
      this.frameCb = v.requestVideoFrameCallback(cb)
    }
    v.addEventListener('loadeddata', this.upload)
    v.addEventListener('seeked', this.upload)
  }

  private upload = () => {
    const v = this.video
    if (v.readyState < 2 || !v.videoWidth) return
    const gl = this.gl
    gl.bindTexture(gl.TEXTURE_2D, this.tex)
    try {
      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, v)
      this.hasFrame = true
      this.dirty = true
    } catch { /* not ready yet */ }
  }

  private draw() {
    const v = this.video as HTMLVideoElement & { requestVideoFrameCallback?: unknown }
    if (!v.requestVideoFrameCallback && !v.paused) this.upload()
    if (!this.dirty && !this.gyro) return
    this.dirty = false
    const c = this.canvas, dpr = Math.min(window.devicePixelRatio || 1, 2)
    const w = Math.round(c.clientWidth * dpr), h = Math.round(c.clientHeight * dpr)
    if (!w || !h) return
    if (c.width !== w || c.height !== h) { c.width = w; c.height = h }
    const gl = this.gl
    gl.viewport(0, 0, w, h)
    if (!this.hasFrame) { gl.clearColor(0, 0, 0, 1); gl.clear(gl.COLOR_BUFFER_BIT); return }
    // The field of view applies to the taller side; on a phone held upright that is the height.
    let right: Vec3, up: Vec3, fwd: Vec3
    if (this.gyro) {
      const q = qmul(axis([0, 1, 0], (-this.gyroYaw * Math.PI) / 180), deviceQuat(this.gyro.alpha, this.gyro.beta, this.gyro.gamma, screenAngle()))
      right = qrot(q, [1, 0, 0]); up = qrot(q, [0, 1, 0]); fwd = qrot(q, [0, 0, -1])
    } else {
      const y = (this.yaw * Math.PI) / 180, p = (this.pitch * Math.PI) / 180
      fwd = [Math.sin(y) * Math.cos(p), Math.sin(p), -Math.cos(y) * Math.cos(p)]
      right = [Math.cos(y), 0, Math.sin(y)]
      up = [right[1] * fwd[2] - right[2] * fwd[1], right[2] * fwd[0] - right[0] * fwd[2], right[0] * fwd[1] - right[1] * fwd[0]]
    }
    // d = right·x + up·y + fwd·1, column-major
    gl.uniformMatrix3fv(this.uView, false, new Float32Array([...right, ...up, ...fwd]))
    gl.uniform2f(this.uAspect, w >= h ? 1 : w / h, w >= h ? h / w : 1)
    gl.uniform1f(this.uFov, (this.fov * Math.PI) / 180)
    gl.drawArrays(gl.TRIANGLES, 0, 6)
  }

  // Dragging by a pixel turns the view by the angle a pixel covers, so what is under the finger
  // stays under it.
  drag(dx: number, dy: number) {
    const per = this.fov / Math.max(this.canvas.clientWidth, this.canvas.clientHeight)
    if (this.gyro) this.gyroYaw = (this.gyroYaw - dx * per) % 360
    else {
      this.yaw = (this.yaw - dx * per) % 360
      this.pitch = Math.max(-89, Math.min(89, this.pitch + dy * per))
    }
    this.dirty = true
  }

  zoom(factor: number) {
    this.fov = Math.max(MIN_FOV, Math.min(MAX_FOV, this.fov * factor))
    this.dirty = true
  }

  get gyroOn() { return this.gyro !== null }

  private onOrient = (e: DeviceOrientationEvent) => {
    if (e.alpha === null || e.beta === null || e.gamma === null) return
    if (!this.gyro) {
      // Keep looking where the drag left off: the phone's heading now is the drag's yaw.
      const q = deviceQuat(e.alpha, e.beta, e.gamma, screenAngle())
      const f = qrot(q, [0, 0, -1])
      this.gyroYaw = this.yaw - (Math.atan2(f[0], -f[2]) * 180) / Math.PI
    }
    this.gyro = { alpha: e.alpha, beta: e.beta, gamma: e.gamma }
  }

  // Turning on motion asks for permission on an iPhone, which must happen in a tap.
  async setGyro(on: boolean): Promise<boolean> {
    if (!on) {
      window.removeEventListener('deviceorientation', this.onOrient)
      if (this.gyro) {
        const q = qmul(axis([0, 1, 0], (-this.gyroYaw * Math.PI) / 180), deviceQuat(this.gyro.alpha, this.gyro.beta, this.gyro.gamma, screenAngle()))
        const f = qrot(q, [0, 0, -1])
        this.yaw = (Math.atan2(f[0], -f[2]) * 180) / Math.PI
        this.pitch = (Math.asin(Math.max(-1, Math.min(1, f[1]))) * 180) / Math.PI
      }
      this.gyro = null
      this.dirty = true
      return false
    }
    const D = DeviceOrientationEvent as unknown as { requestPermission?: () => Promise<string> }
    if (D.requestPermission) {
      try { if ((await D.requestPermission()) !== 'granted') return false } catch { return false }
    }
    window.addEventListener('deviceorientation', this.onOrient)
    return true
  }

  destroy() {
    cancelAnimationFrame(this.raf)
    const v = this.video as HTMLVideoElement & { cancelVideoFrameCallback?: (n: number) => void }
    if (this.frameCb && v.cancelVideoFrameCallback) v.cancelVideoFrameCallback(this.frameCb)
    v.removeEventListener('loadeddata', this.upload)
    v.removeEventListener('seeked', this.upload)
    window.removeEventListener('deviceorientation', this.onOrient)
    this.ro.disconnect()
    this.gl.getExtension('WEBGL_lose_context')?.loseContext()
  }
}

function screenAngle(): number {
  const o = screen.orientation?.angle
  if (typeof o === 'number') return o
  return (window as unknown as { orientation?: number }).orientation ?? 0
}

export const canMotion = () => typeof DeviceOrientationEvent !== 'undefined' && matchMedia('(pointer: coarse)').matches