我正在尝试使用 JavaScript 和 WebGL 实现 Beier 和 Neely 的图像扭曲方法（例如 this 作为示例，并在 here 详细描述）。目标是通过允许用户交互式地在画布上移动点来扭曲面部图像。每个点都会根据所描述的算法操纵图像。
到目前为止，我已经成功创建了一个WebGL程序，允许用户通过单击并拖动图像上的点来与图像进行交互，并且我已经成功设置了基本的图像扭曲效果。
此时，我已经实现了该算法的基本版本，一次允许最多 32 个“点”（计算为线），尽管最终我希望能够使用它来与其他面部跟踪信息进行匹配，所以我想允许数百个/动态数量。
创建了一个带有可移动点的交互式画布。 应用了基本的扭曲效果来操纵这些点周围的图像。
解决了失真效果偏离鼠标光标的问题。
使用适合 WebGL 统一的有限少量点实现了 Beier 和 Neely 的基本变形方法。
剩下的挑战： 创建具有数百个点的效果，可能在 GLSL 纹理中，尽管我正在努力实现它，考虑到制服一次只允许少量的点，而且我还需要允许动态他们的数量。
还请记住，最终目标是能够跟踪一张脸上的点（在不同的项目中完成，使用张量流获取地标）以便能够影响图片的脸部，所以如果有人有更好的建议除了这个之外，对于更好的失真算法，也可以接受。
这是我的代码的当前状态（如果您尝试大图像，请全屏查看）：
<script src="https://cdnjs.cloudflare.com/ajax/libs/gl-matrix/2.8.1/gl-matrix-min.js"></script>
<button id="sl">Select Image</button>
<br>
<button id="editMode">Enter point edit mode</button>
<button id="point" disabled>Add warp point</button>
<br>
<h4>Add at least 3 points</h4><br>
<canvas id="canvas"></canvas>
<style>
.warpPoint {
display:inline-block;
border-radius:50%;
border:1px solid black;
background:gold;
position:absolute;
width: 10px;
height: 10px;
}
.warpPoint:hover {
cursor:pointer;
}
</style>
<script>
let warpPoints = [];
let warpPointsFlat = [];
let warpPointCount = 0;
let isAddingPoints = false;
let img = new Image();
let resolution = vec2.create();
let canvas = document.getElementById('canvas');
let warpPointCountLoc;
let sl = document.getElementById('sl');
let editMode = document.getElementById('editMode');
let point = document.getElementById('point');
let program, gl, imageLoc, resolutionLoc, texture;
let warpPointsLocs = [];
let imageSamplerLoc;
// Initial resolution values
resolution[0] = canvas.width;
resolution[1] = canvas.height;
// Warp point
let warpPoint = vec2.create();
// Functions
function mouseMoveHandler(e, warpPoint, origX, origY, startX, startY) {
let newX = origX + e.clientX - startX;
let newY = origY + e.clientY - startY;
// Account for the canvas' offset from the top-left of the viewport
newX -= canvas.offsetLeft;
newY -= canvas.offsetTop;
warpPoint.setPosition(newX, newY);
var newX_gl;
var newY_gl;
newX_gl = newX / resolution[0]; // Convert to UV coordinates
newY_gl = 1 - newY / resolution[1]; // Convert to UV coordinates
warpPoints[warpPoint.index] = vec4.fromValues(newX_gl, newY_gl, 0.46, 0.2); // Use vec4 and include strength and radius
warpPointsFlat = updateWarpPointsFlat(warpPoints);
render();
}
function WarpPoint(index) {
this.index = index;
this.el = document.createElement("div");
this.el.className = "warpPoint";
// In the WarpPoint setPosition function:
// In the WarpPoint setPosition function:
this.setPosition = function(x, y) {
this.el.style.left = `${x + canvas.offsetLeft}px`;
this.el.style.top = `${y + canvas.offsetTop}px`;
this.x = x / resolution[0]; // Convert to UV coordinates
this.y = 1 - y / resolution[1];
warpPoints[this.index] = vec4.fromValues(this.x, this.y, 0.1, 0.1); // Use vec4 to include z (strength) and w (radius)
warpPointsFlat = updateWarpPointsFlat(warpPoints);
render(); // Make sure to rerender when a warp point is moved
};
document.body.appendChild(this.el);
this.el.onmousedown = (e) => {
var startX = e.clientX;
var startY = e.clientY;
var origX = this.el.offsetLeft;
var origY = this.el.offsetTop;
var mouseMoveHandlerBind = (e) => {
mouseMoveHandler(e, this, origX, origY, startX, startY);
};
var mouseUpHandler = () => {
document.removeEventListener('mousemove', mouseMoveHandlerBind);
document.removeEventListener('mouseup', mouseUpHandler);
};
document.addEventListener('mousemove', mouseMoveHandlerBind);
document.addEventListener('mouseup', mouseUpHandler);
};
}
editMode.onclick = () =>{
if(isAddingPoints) {
point.disabled = true;
isAddingPoints = false;
editMode.innerHTML = "entered WARP mode. Click again to edit"
} else {
point.disabled = false;
isAddingPoints = true;
editMode.innerHTML = "entered EDIT mode. Click again to warp"
}
}
function updateWarpPointsFlat(warpPoints) {
let warpPointsFlat = [];
for(let i = 0; i < warpPoints.length; i++) {
warpPointsFlat.push(warpPoints[i][0]);
warpPointsFlat.push(warpPoints[i][1]);
warpPointsFlat.push(warpPoints[i][2]); // Z
warpPointsFlat.push(warpPoints[i][3]); // W
}
return new Float32Array(warpPointsFlat);
}
point.onclick=addWarpPoint;
function addWarpPoint() {
warpPoints.push(vec4.fromValues(0, 0, 0.1, 0.1));
var warpPoint = new WarpPoint(warpPoints.length - 1); // pass index
let x = (canvas.width / 2);
let y = (canvas.height / 2);
warpPoint.setPosition(x, y);
warpPointCount = warpPoints.length;
warpPointsFlat = updateWarpPointsFlat(warpPoints);
console.log("Warp point added. Current warp points: ", warpPoints);
}
function loadImage(url, callback) {
let image = new Image();
image.src = url;
image.onload = () => callback(image);
}
function createShader(type, source) {
let shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
console.error('Error compiling shader:', gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
function createProgram(vertexShader, fragmentShader) {
let program = gl.createProgram();
gl.attachShader(program, vertexShader);
gl.attachShader(program, fragmentShader);
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
console.error('Error linking program:', gl.getProgramInfoLog(program));
gl.deleteProgram(program);
return null;
}
return program;
}
function createTexture(image) {
let texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
return texture;
}
function resizeCanvas() {
canvas.width = canvas.clientWidth;
canvas.height = canvas.clientHeight;
resolution[0] = canvas.width;
resolution[1] = canvas.height;
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
}
// In the render function:
function render() {
// Clear the canvas
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.useProgram(program);
// Update uniforms
gl.uniform2fv(resolutionLoc, resolution);
// Set up the viewport
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Update uniforms
let warpPointCountUniformLoc = gl.getUniformLocation(program, 'warpPointCount');
let warpPointsUniformLoc = [];
for(let i = 0; i < 32; i++) {
warpPointsUniformLoc[i] = gl.getUniformLocation(program, `warpPoints[${i}]`);
}
gl.uniform1i(warpPointCountUniformLoc, warpPointCount);
// If there are warp points, pass them to the shader
if(warpPointsFlat.length > 0) {
for(let i = 0; i < warpPointCount; i++) {
gl.uniform2fv(warpPointsUniformLoc[i], [warpPointsFlat[i * 4], warpPointsFlat[i * 4 + 1]]);
}
}
// Always bind texture and draw image, regardless of warp point count
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(imageSamplerLoc, 0);
// Draw primitives
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
window.onload = function() {
console.log("hi!")
let canvas = document.getElementById('canvas');
gl = canvas.getContext("webgl")
let vertexShader = createShader(gl.VERTEX_SHADER, `
attribute vec2 position;
varying vec2 uv;
void main() {
uv = position * 0.5 + 0.5;
gl_Position = vec4(position, 0.0, 1.0);
}
`);
let fragmentShader = createShader(gl.FRAGMENT_SHADER, `
precision mediump float;
uniform vec2 resolution;
uniform sampler2D u_image;
uniform vec2 warpPoints[32];
uniform int warpPointCount;
uniform float alpha;
void main() {
vec2 uv = gl_FragCoord.xy / resolution.xy;
vec2 pos = uv;
vec2 displacement = vec2(0.0);
float weightSum = 0.0;
for (int i = 0; i < 32; i += 2) {
if(i >= warpPointCount) break;
vec2 P = warpPoints[i].xy;
vec2 Q = warpPoints[i+1].xy;
float dist_PQ = distance(P, Q);
vec2 dir_PQ = normalize(Q - P);
vec2 X_P = uv - P;
float u = dot(X_P, dir_PQ);
float v = length(cross(vec3(dir_PQ, 0.0), vec3(X_P, 0.0)));
vec2 P_prime = P + u * dir_PQ;
vec2 Q_prime = Q + v * dir_PQ;
vec2 displacement_this_line = P_prime - uv;
float weight_this_line = pow(dist_PQ, alpha) / (0.001 + dot(X_P, X_P));
displacement += weight_this_line * displacement_this_line;
weightSum += weight_this_line;
}
if (weightSum != 0.0) {
displacement /= weightSum;
}
pos += displacement;
vec4 color = texture2D(u_image, clamp(pos, 0.0, 1.0));
gl_FragColor = color;
}
`);
program = createProgram(vertexShader, fragmentShader);
gl.useProgram(program);
gl.uniform2fv(resolutionLoc, resolution);
imageSamplerLoc = gl.getUniformLocation(program, 'u_image');
// Fetch warp point uniform locations
for(let i = 0; i < 10; i++) {
warpPointsLocs[i] = gl.getUniformLocation(program, `warpPoints[${i}]`);
}
let positionLoc = gl.getAttribLocation(program, 'position');
resolutionLoc = gl.getUniformLocation(program, 'resolution');
warpPointCountLoc = gl.getUniformLocation(program, 'warpPointCount');
imageLoc = gl.getUniformLocation(program, 'image');
let positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
-1.0, -1.0,
1.0, -1.0,
-1.0, 1.0,
-1.0, 1.0,
1.0, -1.0,
1.0, 1.0
]), gl.STATIC_DRAW);
gl.enableVertexAttribArray(positionLoc);
gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 0, 0);
};
window.addEventListener('resize', function() {
resizeCanvas();
render();
});
sl.onclick = () => {
var inp = document.createElement("input")
inp.type="file"
inp.onchange=()=>{
var url = URL.createObjectURL(
inp.files[0]
)
img.src=url;
img.onload = () => {
texture = createTexture(img);
canvas.width=img.width;
canvas.height=img.height
resizeCanvas();
render();
};
}
inp.click()
}
</script>坐标是关键！
在您的代码中，您在 screen 和 GL 之间进行了多种约定，但它们并不完全正确。
由于您的
canvascanvas事情变得更加棘手，因为在着色器中，纹理从左下角开始（即在着色器中为 0:0），但对于屏幕上的矩形，它是
offsetLeft:offsetTop + offsetHeight最后一件事是，扭曲公式对我来说看起来不太正确，但我会把它留给你，因为主要要求是让着色器对扭曲点做出正确的反应。
如果这有帮助，请告诉我。
// moved to constants for testing
const STRENGTH = 0.2;
const RADIUS = 50.0 / 300.0;
let warpPoints = [];
let warpPointsFlat = [];
let warpPointCount = 0;
let isEditing = false;
let img = new Image();
let resolution = vec2.create();
let canvas = document.getElementById('canvas');
let warpPointCountLoc;
let sl = document.getElementById('sl');
let editMode = document.getElementById('editMode');
let point = document.getElementById('point');
let program, gl, imageLoc, resolutionLoc, texture;
console.log(888)
// Initial resolution values
resolution[0] = canvas.clientWidth;
resolution[1] = canvas.clientHeight;
// Warp point
let warpPoint = vec2.create();
// Functions
// Functions
function mouseMoveHandler(e, warpPoint, origX, origY, startX, startY) {
let newX = origX + e.clientX - startX;
let newY = origY + e.clientY - startY;
warpPoint.setPosition(newX, newY);
var newX_gl;
var newY_gl;
if (!isEditing) {
let newX_gl = (newX - canvas.offsetLeft) / resolution[0];
let newY_gl = ((canvas.offsetTop + canvas.offsetHeight) - newY) / resolution[1];
warpPoints[warpPoint.index] = vec4.fromValues(newX_gl, newY_gl, STRENGTH, RADIUS); // Use vec4 and include strength and radius
warpPointsFlat = updateWarpPointsFlat(warpPoints);
render();
}
}
function WarpPoint(index) {
this.index = index;
this.el = document.createElement("div");
this.el.className = "warpPoint";
this.setPosition = function (x, y) {
this.el.style.left = `${x}px`;
this.el.style.top = `${y}px`;
let x_gl = (x - canvas.offsetLeft) / resolution[0];
let y_gl = ((canvas.offsetTop + canvas.offsetHeight) - y) / resolution[1];
warpPoints[this.index] = vec4.fromValues(x_gl, y_gl, STRENGTH, RADIUS); // Use vec4 to include z (strength) and w (radius)
warpPointsFlat = updateWarpPointsFlat(warpPoints);
};
document.body.appendChild(this.el);
this.el.onmousedown = (e) => {
var startX = e.clientX;
var startY = e.clientY;
var origX = this.el.offsetLeft;
var origY = this.el.offsetTop;
var mouseMoveHandlerBind = (e) => {
mouseMoveHandler(e, this, origX, origY, startX, startY);
};
var mouseUpHandler = () => {
document.removeEventListener('mousemove', mouseMoveHandlerBind);
document.removeEventListener('mouseup', mouseUpHandler);
};
document.addEventListener('mousemove', mouseMoveHandlerBind);
document.addEventListener('mouseup', mouseUpHandler);
};
}
editMode.onclick = () => {
if (isEditing) {
point.disabled = true;
isEditing = false;
editMode.innerHTML = "entered WARP mode. Click again to edit"
} else {
point.disabled = false;
isEditing = true;
editMode.innerHTML = "entered EDIT mode. Click again to warp"
}
}
function updateWarpPointsFlat(warpPoints) {
let warpPointsFlat = [];
for (let i = 0; i < warpPoints.length; i++) {
warpPointsFlat.push(warpPoints[i][0]);
warpPointsFlat.push(warpPoints[i][1]);
warpPointsFlat.push(STRENGTH); // strength
warpPointsFlat.push(RADIUS); // radius
}
return new Float32Array(warpPointsFlat);
}
point.onclick = addWarpPoint;
function addWarpPoint() {
warpPoints.push(vec4.fromValues(0, 0, STRENGTH, RADIUS));
var warpPoint = new WarpPoint(warpPoints.length - 1); // pass index
let x = canvas.width / 2;
let y = canvas.height / 2;
warpPoint.setPosition(x, y);
warpPointCount = warpPoints.length;
warpPointsFlat = updateWarpPointsFlat(warpPoints);
console.log("Warp point added. Current warp points: ", warpPoints);
}
function loadImage(url, callback) {
let image = new Image();
image.src = url;
image.onload = () => callback(image);
}
function createShader(type, source) {
let shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
console.error('Error compiling shader:', gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
function createProgram(vertexShader, fragmentShader) {
let program = gl.createProgram();
gl.attachShader(program, vertexShader);
gl.attachShader(program, fragmentShader);
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
console.error('Error linking program:', gl.getProgramInfoLog(program));
gl.deleteProgram(program);
return null;
}
return program;
}
function createTexture(image) {
let texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
return texture;
}
function resizeCanvas() {
canvas.width = canvas.clientWidth;
canvas.height = canvas.clientHeight;
resolution[0] = canvas.width;
resolution[1] = canvas.height;
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
}
function render() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(program);
gl.uniform2fv(resolutionLoc, resolution);
// If warp points are present, update uniform data
if (warpPointsFlat.length > 0) {
for (let i = 0; i < warpPointCount; i++) {
let offset = i * 4;
let warpPoint_i_Location = gl.getUniformLocation(program, 'warpPoints[' + i + ']');
gl.uniform4fv(warpPoint_i_Location, warpPointsFlat.subarray(offset, offset + 4));
}
}
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(imageLoc, 0);
gl.uniform1i(warpPointCountLoc, warpPointCount);
gl.drawArrays(gl.TRIANGLES, 0, 6);
console.log("(at end of render..)Rendering with resolution: ", resolution);
console.log("Rendering with warp points: ", warpPointsFlat);
console.log("Rendering with warp point count (end of render..): ", warpPointCount);
}
console.log("ok")
window.onload = function () {
console.log("hi!")
let canvas = document.getElementById('canvas');
gl = canvas.getContext("webgl")
console.log(gl, 22)
let vertexShader = createShader(gl.VERTEX_SHADER, `
attribute vec2 position;
varying vec2 uv;
void main() {
uv = position * 0.5 + 0.5;
gl_Position = vec4(position, 0.0, 1.0);
}
`);
let fragmentShader = createShader(gl.FRAGMENT_SHADER, `
precision mediump float;
const int MAX_WARP_POINTS = 10;
uniform vec2 resolution;
uniform vec4 warpPoints[MAX_WARP_POINTS];
uniform int warpPointCount;
uniform sampler2D image;
varying vec2 uv;
void main() {
vec2 pos = uv;
vec2 newPos = pos;
for(int i = 0; i < MAX_WARP_POINTS; i++) {
if(i >= warpPointCount) {
break;
}
vec2 warpPoint = warpPoints[i].xy;
vec2 offset = warpPoint - pos;
float dist = length(offset);
vec2 direction = offset / dist;
float strength = warpPoints[i].z;
float radius = warpPoints[i].w;
if (dist < radius) {
newPos -= strength * direction * (1.0 - smoothstep(0.0, radius, dist));
}
}
gl_FragColor = texture2D(image, newPos);
}
`);
program = createProgram(vertexShader, fragmentShader);
let positionLoc = gl.getAttribLocation(program, 'position');
resolutionLoc = gl.getUniformLocation(program, 'resolution');
warpPointCountLoc = gl.getUniformLocation(program, 'warpPointCount');
imageLoc = gl.getUniformLocation(program, 'image');
let positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
-1.0, -1.0,
1.0, -1.0,
-1.0, 1.0,
-1.0, 1.0,
1.0, -1.0,
1.0, 1.0
]), gl.STATIC_DRAW);
gl.enableVertexAttribArray(positionLoc);
gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 0, 0);
};
window.addEventListener('resize', function () {
resizeCanvas();
render();
});
sl.onclick = () => {
console.log(2)
var inp = document.createElement("input")
inp.type = "file"
inp.onchange = () => {
var url = URL.createObjectURL(
inp.files[0]
)
img.src = url;
img.onload = () => {
texture = createTexture(img);
resizeCanvas();
render();
};
}
inp.click()
} <script src="https://cdnjs.cloudflare.com/ajax/libs/gl-matrix/2.8.1/gl-matrix-min.js"></script>
<button id="sl">Select Image</button>
<br>
<button id="editMode">Enter point edit mode</button>
<button id="point" disabled>Add warp point</button>
<br>
<canvas id="canvas"></canvas>
<style>
.warpPoint {
display: inline-block;
border-radius: 50%;
border: 1px solid black;
background: gold;
position: absolute;
width: 10px;
height: 10px;
}
.warpPoint:hover {
cursor: pointer;
}
</style>