/** * @license * SPDX-License-Identifier: Apache-2.0 */ // tslint:disable:organize-imports // tslint:disable:ban-malformed-import-paths // tslint:disable:no-new-decorators import {LitElement, css, html} from 'lit'; import {customElement, property} from 'lit/decorators.js'; import {Analyser} from './analyser'; import * as THREE from 'three'; import {EXRLoader} from 'three/addons/loaders/EXRLoader.js'; import {EffectComposer} from 'three/addons/postprocessing/EffectComposer.js'; import {RenderPass} from 'three/addons/postprocessing/RenderPass.js'; import {ShaderPass} from 'three/addons/postprocessing/ShaderPass.js'; import {UnrealBloomPass} from 'three/addons/postprocessing/UnrealBloomPass.js'; import {FXAAShader} from 'three/addons/shaders/FXAAShader.js'; import {fs as backdropFS, vs as backdropVS} from './backdrop-shader'; import {vs as sphereVS} from './sphere-shader'; /** * 3D live audio visual. */ @customElement('gdm-live-audio-visuals-3d') export class GdmLiveAudioVisuals3D extends LitElement { private inputAnalyser!: Analyser; private outputAnalyser!: Analyser; private camera!: THREE.PerspectiveCamera; private backdrop!: THREE.Mesh; private composer!: EffectComposer; private sphere!: THREE.Mesh; private prevTime = 0; private rotation = new THREE.Vector3(0, 0, 0); private _outputNode!: AudioNode; @property() set outputNode(node: AudioNode) { this._outputNode = node; this.outputAnalyser = new Analyser(this._outputNode); } get outputNode() { return this._outputNode; } private _inputNode!: AudioNode; @property() set inputNode(node: AudioNode) { this._inputNode = node; this.inputAnalyser = new Analyser(this._inputNode); } get inputNode() { return this._inputNode; } private canvas!: HTMLCanvasElement; static styles = css` canvas { width: 100% !important; height: 100% !important; position: absolute; inset: 0; image-rendering: pixelated; } `; connectedCallback() { super.connectedCallback(); } private init() { const scene = new THREE.Scene(); scene.background = new THREE.Color(0x100c14); const backdrop = new THREE.Mesh( new THREE.IcosahedronGeometry(10, 5), new THREE.RawShaderMaterial({ uniforms: { resolution: {value: new THREE.Vector2(1, 1)}, rand: {value: 0}, }, vertexShader: backdropVS, fragmentShader: backdropFS, glslVersion: THREE.GLSL3, }), ); backdrop.material.side = THREE.BackSide; scene.add(backdrop); this.backdrop = backdrop; const camera = new THREE.PerspectiveCamera( 75, window.innerWidth / window.innerHeight, 0.1, 1000, ); camera.position.set(2, -2, 5); this.camera = camera; const renderer = new THREE.WebGLRenderer({ canvas: this.canvas, antialias: !true, }); renderer.setSize(window.innerWidth, window.innerHeight); renderer.setPixelRatio(window.devicePixelRatio / 1); const geometry = new THREE.IcosahedronGeometry(1, 10); new EXRLoader().load('piz_compressed.exr', (texture: THREE.Texture) => { texture.mapping = THREE.EquirectangularReflectionMapping; const exrCubeRenderTarget = pmremGenerator.fromEquirectangular(texture); sphereMaterial.envMap = exrCubeRenderTarget.texture; sphere.visible = true; }); const pmremGenerator = new THREE.PMREMGenerator(renderer); pmremGenerator.compileEquirectangularShader(); const sphereMaterial = new THREE.MeshStandardMaterial({ color: 0x000010, metalness: 0.5, roughness: 0.1, emissive: 0x000010, emissiveIntensity: 1.5, }); sphereMaterial.onBeforeCompile = (shader) => { shader.uniforms.time = {value: 0}; shader.uniforms.inputData = {value: new THREE.Vector4()}; shader.uniforms.outputData = {value: new THREE.Vector4()}; sphereMaterial.userData.shader = shader; shader.vertexShader = sphereVS; }; const sphere = new THREE.Mesh(geometry, sphereMaterial); scene.add(sphere); sphere.visible = false; this.sphere = sphere; const renderPass = new RenderPass(scene, camera); const bloomPass = new UnrealBloomPass( new THREE.Vector2(window.innerWidth, window.innerHeight), 5, 0.5, 0, ); const fxaaPass = new ShaderPass(FXAAShader); const composer = new EffectComposer(renderer); composer.addPass(renderPass); // composer.addPass(fxaaPass); composer.addPass(bloomPass); this.composer = composer; function onWindowResize() { camera.aspect = window.innerWidth / window.innerHeight; camera.updateProjectionMatrix(); const dPR = renderer.getPixelRatio(); const w = window.innerWidth; const h = window.innerHeight; backdrop.material.uniforms.resolution.value.set(w * dPR, h * dPR); renderer.setSize(w, h); composer.setSize(w, h); fxaaPass.material.uniforms['resolution'].value.set( 1 / (w * dPR), 1 / (h * dPR), ); } window.addEventListener('resize', onWindowResize); onWindowResize(); this.animation(); } private animation() { requestAnimationFrame(() => this.animation()); this.inputAnalyser.update(); this.outputAnalyser.update(); const t = performance.now(); const dt = (t - this.prevTime) / (1000 / 60); this.prevTime = t; const backdropMaterial = this.backdrop.material as THREE.RawShaderMaterial; const sphereMaterial = this.sphere.material as THREE.MeshStandardMaterial; backdropMaterial.uniforms.rand.value = Math.random() * 10000; if (sphereMaterial.userData.shader) { this.sphere.scale.setScalar( 1 + (0.2 * this.outputAnalyser.data[1]) / 255, ); const f = 0.001; this.rotation.x += (dt * f * 0.5 * this.outputAnalyser.data[1]) / 255; this.rotation.z += (dt * f * 0.5 * this.inputAnalyser.data[1]) / 255; this.rotation.y += (dt * f * 0.25 * this.inputAnalyser.data[2]) / 255; this.rotation.y += (dt * f * 0.25 * this.outputAnalyser.data[2]) / 255; const euler = new THREE.Euler( this.rotation.x, this.rotation.y, this.rotation.z, ); const quaternion = new THREE.Quaternion().setFromEuler(euler); const vector = new THREE.Vector3(0, 0, 5); vector.applyQuaternion(quaternion); this.camera.position.copy(vector); this.camera.lookAt(this.sphere.position); sphereMaterial.userData.shader.uniforms.time.value += (dt * 0.1 * this.outputAnalyser.data[0]) / 255; sphereMaterial.userData.shader.uniforms.inputData.value.set( (1 * this.inputAnalyser.data[0]) / 255, (0.1 * this.inputAnalyser.data[1]) / 255, (10 * this.inputAnalyser.data[2]) / 255, 0, ); sphereMaterial.userData.shader.uniforms.outputData.value.set( (2 * this.outputAnalyser.data[0]) / 255, (0.1 * this.outputAnalyser.data[1]) / 255, (10 * this.outputAnalyser.data[2]) / 255, 0, ); } this.composer.render(); } protected firstUpdated() { this.canvas = this.shadowRoot!.querySelector('canvas') as HTMLCanvasElement; this.init(); } protected render() { return html``; } } declare global { interface HTMLElementTagNameMap { 'gdm-live-audio-visuals-3d': GdmLiveAudioVisuals3D; } }