Omniverse Realtime Viewer

SHM ring buffer slot (BGRA8, pitch-aligned)
  -> N-API addon: memcpy to SharedArrayBuffer on a libuv worker thread
  -> Electron IPC/preload: SAB handle crosses process boundary without copying
  -> React renderer: Uint8Array view of the SAB pixel payload
  -> WebGL: texImage2D first frame, texSubImage2D later frames
  -> full-viewport textured quad
  -> canvas sized with object-fit: contain for letterbox

byte 0..3    uint32 width
byte 4..7    uint32 height
byte 8..15   uint64 sequence
byte 16..N   pixel bytes

const HEADER_BYTES = 16;

interface FrameHeader { width: number; height: number; sequence: bigint }

function readFrameHeader(buffer: SharedArrayBuffer): FrameHeader {
  const view = new DataView(buffer, 0, HEADER_BYTES);
  return { width: view.getUint32(0, true), height: view.getUint32(4, true), sequence: view.getBigUint64(8, true) };
}

session.defaultSession.webRequest.onHeadersReceived((details, callback) => {
  callback({
    responseHeaders: {
      ...details.responseHeaders,
      'Cross-Origin-Opener-Policy': ['same-origin'],
      'Cross-Origin-Embedder-Policy': ['require-corp'],
    },
  });
});

const win = new BrowserWindow({
  webPreferences: { preload: PRELOAD_PATH, contextIsolation: true, nodeIntegration: false, sandbox: true },
});

contextBridge.exposeInMainWorld('shmFrames', {
  getFrameBuffer: () => ipcRenderer.sendSync('shm:get-frame-buffer'),
  getFrameMetadata: () => ipcRenderer.sendSync('shm:get-frame-metadata'),
  onFrameAvailable: (callback: () => void) => {
    const listener = () => callback();
    ipcRenderer.on('shm:frame-available', listener);
    return () => ipcRenderer.removeListener('shm:frame-available', listener);
  },
});

slot = ring.newest_complete_slot()
copy header: width, height, sequence
for row in 0..height:
  memcpy(sab.pixels + row * width * 4, slot.pixels + row * slot.pitch_bytes, width * 4)
notify renderer: shm:frame-available

attribute vec2 a_position;
attribute vec2 a_texCoord;

varying vec2 v_texCoord;

void main() {
  v_texCoord = a_texCoord;
  gl_Position = vec4(a_position, 0.0, 1.0); // viewport clip-space position
}

precision mediump float;

uniform sampler2D u_frame;

varying vec2 v_texCoord;

void main() {
  gl_FragColor = texture2D(u_frame, v_texCoord);
}

const QUAD = new Float32Array([
  -1, -1, 0, 1,  1, -1, 1, 1,
  -1,  1, 0, 0,  1,  1, 1, 0,
]);

function createTexture(gl: WebGLRenderingContext): WebGLTexture {
  const texture = gl.createTexture();
  if (!texture) throw new Error('Failed to create texture');
  gl.bindTexture(gl.TEXTURE_2D, texture);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
  gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
  return texture;
}

function uploadFrame(
  gl: WebGLRenderingContext,
  texture: WebGLTexture,
  format: number,
  pixels: Uint8Array,
  width: number,
  height: number,
  previous: { width: number; height: number },
) {
  gl.bindTexture(gl.TEXTURE_2D, texture);
  if (previous.width !== width || previous.height !== height) {
    gl.texImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, format, gl.UNSIGNED_BYTE, pixels);
    previous.width = width;
    previous.height = height;
  } else {
    gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, width, height, format, gl.UNSIGNED_BYTE, pixels);
  }
}

const ext = gl.getExtension('EXT_texture_format_BGRA8888') as { BGRA_EXT: number } | null;
const uploadFormat = metadata.format === 'BGRA8' && ext ? ext.BGRA_EXT : gl.RGBA;

function bgraToRgba(source: Uint8Array, scratch: Uint8Array): Uint8Array {
  const src = new Uint32Array(source.buffer, source.byteOffset, source.byteLength / 4);
  const dst = new Uint32Array(scratch.buffer, scratch.byteOffset, scratch.byteLength / 4);
  for (let i = 0; i < src.length; i += 1) {
    const p = src[i];
    dst[i] = (p & 0xff00ff00) | ((p & 0xff) << 16) | ((p >>> 16) & 0xff);
  }
  return scratch;
}

import { useEffect, useRef, useState } from 'react';

declare global {
  interface Window {
    shmFrames: {
      getFrameBuffer(): SharedArrayBuffer | null;
      getFrameMetadata(): { pitchBytes?: number; format: 'BGRA8' | 'RGBA8' };
      onFrameAvailable(callback: () => void): () => void;
    };
  }
}

export function useWebGLCanvas(canvasRef: React.RefObject<HTMLCanvasElement>) {
  const lastSequence = useRef<bigint>(-1n);
  const needsDraw = useRef(true);
  const scratch = useRef<Uint8Array | null>(null);
  const [error, setError] = useState<string | null>(null);

  useEffect(() => {
    const canvas = canvasRef.current;
    if (!canvas) return;

    const gl = canvas.getContext('webgl', {
      alpha: false,
      antialias: false,
      depth: false,
      stencil: false,
      preserveDrawingBuffer: false,
    });
    if (!gl) {
      setError('WebGL unavailable');
      return;
    }

    const metadata = window.shmFrames.getFrameMetadata();
    const ext = gl.getExtension('EXT_texture_format_BGRA8888') as { BGRA_EXT: number } | null;
    const nativeBgra = metadata.format === 'BGRA8' && !!ext;
    const uploadFormat = nativeBgra ? ext!.BGRA_EXT : gl.RGBA;
    const program = createProgram(gl, VERTEX_SHADER_SOURCE, FRAGMENT_SHADER_SOURCE);
    const quad = createQuadBuffer(gl, program, QUAD);
    const texture = createTexture(gl);
    const previous = { width: 0, height: 0 };

    const unsubscribe = window.shmFrames.onFrameAvailable(() => {
      needsDraw.current = true;
    });

    let raf = 0;
    const draw = () => {
      raf = requestAnimationFrame(draw);
      if (!needsDraw.current) return;

      const sab = window.shmFrames.getFrameBuffer();
      if (!sab) return;

      const header = readFrameHeader(sab);
      if (!header.width || !header.height || header.sequence === lastSequence.current) return;

      if (canvas.width !== header.width || canvas.height !== header.height) {
        canvas.width = header.width;
        canvas.height = header.height;
        gl.viewport(0, 0, header.width, header.height);
      }

      const byteLength = header.width * header.height * 4;
      const source = new Uint8Array(sab, HEADER_BYTES, byteLength);
      let pixels = source;
      if (metadata.format === 'BGRA8' && !nativeBgra) {
        if (!scratch.current || scratch.current.byteLength !== byteLength) {
          scratch.current = new Uint8Array(byteLength);
        }
        pixels = bgraToRgba(source, scratch.current);
      }

      gl.useProgram(program);
      gl.bindBuffer(gl.ARRAY_BUFFER, quad);
      uploadFrame(gl, texture, uploadFormat, pixels, header.width, header.height, previous);
      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);

      lastSequence.current = header.sequence;
      needsDraw.current = false;
    };

    const onLost = (event: Event) => { event.preventDefault(); setError('WebGL context lost'); };
    canvas.addEventListener('webglcontextlost', onLost);
    raf = requestAnimationFrame(draw);

    return () => {
      cancelAnimationFrame(raf);
      unsubscribe();
      canvas.removeEventListener('webglcontextlost', onLost);
      gl.deleteTexture(texture);
      gl.deleteBuffer(quad);
      gl.deleteProgram(program);
    };
  }, [canvasRef]);

  return { error };
}

.viewportShell { width: 100%; height: 100%; overflow: hidden; background: #0b0d10; }
.viewportCanvas { display: block; width: 100%; height: 100%; object-fit: contain; }

function clientToFramePoint(event: React.PointerEvent<HTMLElement>, frameWidth: number, frameHeight: number) {
  const rect = event.currentTarget.getBoundingClientRect();
  const scale = Math.min(rect.width / frameWidth, rect.height / frameHeight);
  const contentWidth = frameWidth * scale;
  const contentHeight = frameHeight * scale;
  const offsetX = (rect.width - contentWidth) / 2;
  const offsetY = (rect.height - contentHeight) / 2;
  const x = event.clientX - rect.left - offsetX;
  const y = event.clientY - rect.top - offsetY;
  if (x < 0 || y < 0 || x >= contentWidth || y >= contentHeight) return null;
  return { x: Math.floor((x / contentWidth) * frameWidth), y: Math.floor((y / contentHeight) * frameHeight) };
}