From cc3bdd1b0a5e48d3512df5c048418dffd31935de Mon Sep 17 00:00:00 2001 From: Pierre HUBERT Date: Sat, 30 Jan 2021 19:13:21 +0100 Subject: [PATCH] Start to blur video background --- .../tensorflow-models/body-pix-2.0.js | 18 ++++++ assets/3rdparty/tfjs/tfjs-1.2.min.js | 3 + assets/js/components/calls/window.js | 57 +++++++++++++++++++ system/config/dev.config.php | 6 +- 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 assets/3rdparty/tensorflow-models/body-pix-2.0.js create mode 100644 assets/3rdparty/tfjs/tfjs-1.2.min.js diff --git a/assets/3rdparty/tensorflow-models/body-pix-2.0.js b/assets/3rdparty/tensorflow-models/body-pix-2.0.js new file mode 100644 index 00000000..fdc628b5 --- /dev/null +++ b/assets/3rdparty/tensorflow-models/body-pix-2.0.js @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("@tensorflow/tfjs-core"),require("@tensorflow/tfjs-converter")):"function"==typeof define&&define.amd?define(["exports","@tensorflow/tfjs-core","@tensorflow/tfjs-converter"],e):e(t.bodyPix={},t.tf,t.tf)}(this,function(t,e,n){"use strict";var r=function(t,e){return(r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n])})(t,e)};function o(t,e){function n(){this.constructor=t}r(t,e),t.prototype=null===e?Object.create(e):(n.prototype=e.prototype,new n)}var i=function(){return(i=Object.assign||function(t){for(var e,n=1,r=arguments.length;n0&&o[o.length-1])&&(6===i[0]||2===i[0])){a=0;continue}if(3===i[0]&&(!o||i[1]>o[0]&&i[1]n?n:t}function y(t,e){return{x:t.x+e.x,y:t.y+e.y}}function b(t,e,n){void 0===n&&(n=.3);for(var r=0,o=0,i=0;in&&(o+=1,r+=Math.pow(t[i].x-e.keypoints[i].position.x,2)+Math.pow(t[i].y-e.keypoints[i].position.y,2));return 0===o?r=1/0:r/=o,r}function x(t,e,n,r,o,i,a){for(var s=a[0],u=a[1],d=n(t),f=d.y*r+d.x,l=o[h*(2*f)+e],c=o[h*(2*f+1)+e],p=t.y+l,m=t.x+c,g=0;g "+f+") {\n numKpt = numKpt + 1;\n curDistSum = curDistSum + dist(x, y, poseX, poseY);\n }\n }\n if (numKpt > 0 && curDistSum / float(numKpt) < minDist) {\n minDist = curDistSum / float(numKpt);\n iMin = i;\n }\n }\n return iMin;\n }\n\n void main() {\n ivec2 coords = getOutputCoords();\n int nearestPose = findNearestPose(coords[0], coords[1]);\n setOutput(float(nearestPose));\n }\n "};return e.backend().compileAndRun(D,[t,S,T])}function _(){return"webgl"===e.getBackend()}function E(t,n,r,o,i,u,d,f,l,c,h,p){var g=d[0],v=d[1];return void 0===l&&(l=.2),void 0===c&&(c=8),void 0===h&&(h=.3),void 0===p&&(p=10),a(this,void 0,void 0,function(){var a,d,w,y,b;return s(this,function(s){switch(s.label){case 0:return a=r.filter(function(t){return t.score>=l}),_()?(w=e.tidy(function(){var r=k(t,n,a,o,i,u,[g,v],f,c,h,p);return a.map(function(t,n){return function(t,n){return e.tidy(function(){return t.equal(e.scalar(n)).toInt()})}(r,n)})}),[4,Promise.all(w.map(function(t){return t.data()}))]):[3,2];case 1:return d=s.sent(),w.forEach(function(t){return t.dispose()}),[3,5];case 2:return[4,t.data()];case 3:return y=s.sent(),[4,n.data()];case 4:b=s.sent(),d=function(t,e,n,r,o,i,a,s,u,d){var f=a[0],l=a[1];void 0===d&&(d=5);for(var c=n.map(function(t){return new Uint8Array(r*o).fill(0)}),h=s.top,p=s.left,g=m([r,o],[f,l],s),v=g[0],w=g[1],y=M([f,l],i)[0],b=0;b=0&&(c[_][k]=1)}}return c}(y,b,a,o,i,u,[g,v],f,c),s.label=5;case 5:return[2,d.map(function(t,e){return{data:t,pose:a[e],width:i,height:o}})]}})})}function P(t,n,r,o,i,u,d,f,l,c,h,p,g){var v=f[0],w=f[1];return void 0===c&&(c=.2),void 0===h&&(h=8),void 0===p&&(p=.3),void 0===g&&(g=10),a(this,void 0,void 0,function(){var a,f,y,b,x,E;return s(this,function(s){switch(s.label){case 0:return a=o.filter(function(t){return t.score>=c}),_()?(y=e.tidy(function(){var o=k(t,n,a,i,u,d,[v,w],l,h,p,g);return a.map(function(t,n){return function(t,n,r){return e.tidy(function(){return t.equal(e.scalar(r)).toInt().mul(n.add(1)).sub(1)})}(o,r,n)})}),[4,Promise.all(y.map(function(t){return t.data()}))]):[3,2];case 1:return f=s.sent(),y.forEach(function(t){return t.dispose()}),[3,6];case 2:return[4,t.data()];case 3:return b=s.sent(),[4,n.data()];case 4:return x=s.sent(),[4,r.data()];case 5:E=s.sent(),f=function(t,e,n,r,o,i,a,s,u,d,f){var l=s[0],c=s[1];void 0===f&&(f=5);for(var h=r.map(function(t){return new Int32Array(o*i).fill(-1)}),p=u.top,g=u.left,v=m([o,i],[l,c],u),w=v[0],y=v[1],b=M([l,c],a)[0],x=0;x=0&&(h[E][_]=n[_])}}return h}(b,x,E,a,i,u,d,[v,w],l,h),s.label=6;case 6:return[2,f.map(function(t,e){return{pose:a[e],data:t,height:i,width:u}})]}})})}function O(t){return Math.floor(t/2)}var I=function(){function t(t,e){this.priorityQueue=new Array(t),this.numberOfElements=-1,this.getElementValue=e}return t.prototype.enqueue=function(t){this.priorityQueue[++this.numberOfElements]=t,this.swim(this.numberOfElements)},t.prototype.dequeue=function(){var t=this.priorityQueue[0];return this.exchange(0,this.numberOfElements--),this.sink(0),this.priorityQueue[this.numberOfElements+1]=null,t},t.prototype.empty=function(){return-1===this.numberOfElements},t.prototype.size=function(){return this.numberOfElements+1},t.prototype.all=function(){return this.priorityQueue.slice(0,this.numberOfElements+1)},t.prototype.max=function(){return this.priorityQueue[0]},t.prototype.swim=function(t){for(;t>0&&this.less(O(t),t);)this.exchange(t,O(t)),t=O(t)},t.prototype.sink=function(t){for(;2*t<=this.numberOfElements;){var e=2*t;if(ee){d=!1;break}if(!d)break}return d}var R=[["nose","leftEye"],["leftEye","leftEar"],["nose","rightEye"],["rightEye","rightEar"],["nose","leftShoulder"],["leftShoulder","leftElbow"],["leftElbow","leftWrist"],["leftShoulder","leftHip"],["leftHip","leftKnee"],["leftKnee","leftAnkle"],["nose","rightShoulder"],["rightShoulder","rightElbow"],["rightElbow","rightWrist"],["rightShoulder","rightHip"],["rightHip","rightKnee"],["rightKnee","rightAnkle"]].map(function(t){var e=t[0],n=t[1];return[p[e],p[n]]}),H=R.map(function(t){return t[1]}),T=R.map(function(t){return t[0]});function B(t,e,n,r){return{y:w(Math.round(t.y/e),0,n-1),x:w(Math.round(t.x/e),0,r-1)}}function F(t,e,n,r,o,i,a,s){void 0===s&&(s=2);for(var u=r.shape,d=u[0],f=u[1],l=function(t,e,n){var r=n.shape[2]/2;return{y:n.get(e.y,e.x,t),x:n.get(e.y,e.x,r+t)}}(t,B(e.position,i,d,f),a),h=y(e.position,l),p=0;p=0;--h){var p=H[h],m=T[h];u[p]&&!u[m]&&(u[m]=F(h,u[p],m,e,n,r,i))}for(h=0;h=G,function(){return"inputResolution must be a string or number between "+G+" and "+J+", but was "+t}),t}(t);return[U(o*a,n),U(i*a,n)]}function $(t,n,r,o,i){var a=n[0],s=n[1],u=r[0],d=r[1],f=o[0],l=f[0],c=f[1],h=o[1],p=h[0],m=h[1];return void 0===i&&(i=!1),e.tidy(function(){var n=t.resizeBilinear([u,d],!0);return i&&(n=n.sigmoid()),function(t,n,r){var o=n[0],i=n[1],a=r[0],s=a[0],u=a[1],d=r[1],f=d[0],l=d[1];return e.tidy(function(){return e.image.cropAndResize(t.expandDims(),[[s/(o+s+u-1),f/(i+f+l-1),(s+o-1)/(o+s+u-1),(f+i-1)/(i+f+l-1)]],[0],[o,i]).squeeze([0])})}(n,[a,s],[[l,c],[p,m]])})}function tt(t,n){var r=n[0],o=n[1],i=Q(t),a=i[0],s=i[1],u=o/r,d=[0,0,0,0],f=d[0],l=d[1],c=d[2],h=d[3];return s/a1)throw new Error("segmentationThreshold "+e+". Should be in range [0.0, 1.0]");if(n<=0)throw new Error("Invalid maxDetections "+n+". Should be > 0");if(r<0||r>1)throw new Error("Invalid scoreThreshold "+r+". Should be in range [0.0, 1.0]");if(o<=0)throw new Error("Invalid nmsRadius "+o+".")}function ct(t){var e=t.segmentationThreshold,n=t.maxDetections,r=t.scoreThreshold,o=t.nmsRadius,i=t.minKeypointScore,a=t.refineSteps;if(e<0||e>1)throw new Error("segmentationThreshold "+e+". Should be in range [0.0, 1.0]");if(n<=0)throw new Error("Invalid maxDetections "+n+". Should be > 0");if(r<0||r>1)throw new Error("Invalid scoreThreshold "+r+". Should be in range [0.0, 1.0]");if(o<=0)throw new Error("Invalid nmsRadius "+o+".");if(i<0||i>1)throw new Error("Invalid minKeypointScore "+i+".Should be in range [0.0, 1.0]");if(a<=0||a>20)throw new Error("Invalid refineSteps "+a+".Should be in range [1, 20]")}var ht=function(){function t(t){this.baseModel=t}return t.prototype.predictForPersonSegmentation=function(t){var e=this.baseModel.predict(t);return{segmentLogits:e.segmentation,heatmapScores:e.heatmapScores,offsets:e.offsets,displacementFwd:e.displacementFwd,displacementBwd:e.displacementBwd}},t.prototype.predictForPersonSegmentationAndPart=function(t){var e=this.baseModel.predict(t);return{segmentLogits:e.segmentation,partHeatmapLogits:e.partHeatmaps,heatmapScores:e.heatmapScores,offsets:e.offsets,displacementFwd:e.displacementFwd,displacementBwd:e.displacementBwd}},t.prototype.predictForMultiPersonInstanceSegmentationAndPart=function(t){var e=this.baseModel.predict(t);return{segmentLogits:e.segmentation,longOffsets:e.longOffsets,heatmapScores:e.heatmapScores,offsets:e.offsets,displacementFwd:e.displacementFwd,displacementBwd:e.displacementBwd,partHeatmaps:e.partHeatmaps}},t.prototype.segmentPersonActivation=function(t,n,r){var o=this;void 0===r&&(r=.5);var i=Q(t),a=i[0],s=i[1],u=Z(n,this.baseModel.outputStride,[a,s]),f=tt(t,u),l=f.resized,c=f.padding,h=e.tidy(function(){var t=o.predictForPersonSegmentation(l),e=t.segmentLogits,n=t.heatmapScores,i=t.offsets,u=t.displacementFwd,f=t.displacementBwd,h=l.shape,p=h[0],m=h[1];return{segmentation:d($(e,[a,s],[p,m],[[c.top,c.bottom],[c.left,c.right]],!0).squeeze(),r),heatmapScores:n,offsets:i,displacementFwd:u,displacementBwd:f}}),p=h.segmentation,m=h.heatmapScores,g=h.offsets,v=h.displacementFwd,w=h.displacementBwd;return l.dispose(),{segmentation:p,heatmapScores:m,offsets:g,displacementFwd:v,displacementBwd:w,padding:c,internalResolutionHeightAndWidth:u}},t.prototype.segmentPerson=function(t,e){return void 0===e&&(e=dt),a(this,void 0,void 0,function(){var n,r,o,a,u,d,f,l,c,h,p,m,g,v,w,y,b,x;return s(this,function(s){switch(s.label){case 0:return lt(e=i({},dt,e)),n=this.segmentPersonActivation(t,e.internalResolution,e.segmentationThreshold),r=n.segmentation,o=n.heatmapScores,a=n.offsets,u=n.displacementFwd,d=n.displacementBwd,f=n.padding,l=n.internalResolutionHeightAndWidth,c=r.shape,h=c[0],p=c[1],[4,r.data()];case 1:return m=s.sent(),r.dispose(),[4,et([o,a,u,d])];case 2:return g=s.sent(),v=g[0],w=g[1],y=g[2],b=g[3],x=rt(x=C(v,w,y,b,this.baseModel.outputStride,e.maxDetections,e.scoreThreshold,e.nmsRadius),[h,p],l,f,!1),o.dispose(),a.dispose(),u.dispose(),d.dispose(),[2,{height:h,width:p,data:m,allPoses:x}]}})})},t.prototype.segmentMultiPerson=function(t,n){return void 0===n&&(n=ft),a(this,void 0,void 0,function(){var r,o,a,u,f,l,c,h,p,m,g,v,w,y,b,x,S,M,k,_,P,O=this;return s(this,function(s){switch(s.label){case 0:return ct(n=i({},ft,n)),r=Q(t),o=r[0],a=r[1],u=Z(n.internalResolution,this.baseModel.outputStride,[o,a]),f=tt(t,u),l=f.resized,c=f.padding,h=e.tidy(function(){var t,e=O.predictForMultiPersonInstanceSegmentationAndPart(l),r=e.segmentLogits,i=e.longOffsets,s=e.heatmapScores,f=e.offsets,h=e.displacementFwd,p=e.displacementBwd;return t=i,{segmentation:d($(r,[o,a],u,[[c.top,c.bottom],[c.left,c.right]],!0).squeeze(),n.segmentationThreshold),longOffsets:t,heatmapScoresRaw:s,offsetsRaw:f,displacementFwdRaw:h,displacementBwdRaw:p}}),p=h.segmentation,m=h.longOffsets,g=h.heatmapScoresRaw,v=h.offsetsRaw,w=h.displacementFwdRaw,y=h.displacementBwdRaw,[4,et([g,v,w,y])];case 1:return b=s.sent(),x=b[0],S=b[1],M=b[2],k=b[3],_=rt(_=C(x,S,M,k,this.baseModel.outputStride,n.maxDetections,n.scoreThreshold,n.nmsRadius),[o,a],u,c,!1),[4,E(p,m,_,o,a,this.baseModel.outputStride,u,c,n.scoreThreshold,n.refineSteps,n.minKeypointScore,n.maxDetections)];case 2:return P=s.sent(),l.dispose(),p.dispose(),m.dispose(),g.dispose(),v.dispose(),w.dispose(),y.dispose(),[2,P]}})})},t.prototype.segmentPersonPartsActivation=function(t,n,r){var o=this;void 0===r&&(r=.5);var i=Q(t),a=i[0],s=i[1],f=Z(n,this.baseModel.outputStride,[a,s]),l=tt(t,f),c=l.resized,h=l.padding,p=e.tidy(function(){var t=o.predictForPersonSegmentationAndPart(c),n=t.segmentLogits,i=t.partHeatmapLogits,f=t.heatmapScores,l=t.offsets,p=t.displacementFwd,m=t.displacementBwd,g=c.shape,v=g[0],w=g[1],y=$(n,[a,s],[v,w],[[h.top,h.bottom],[h.left,h.right]],!0),b=$(i,[a,s],[v,w],[[h.top,h.bottom],[h.left,h.right]],!0);return{partSegmentation:function(t,n){var r=n.shape,o=r[0],i=r[1],a=r[2];return e.tidy(function(){var r,s,d=u(n),f=e.range(0,a,1,"int32").expandDims(1),l=d.matMul(f).toInt().reshape([o,i]).add(e.scalar(1,"int32"));return(r=l,s=t,r.mul(s)).sub(e.scalar(1,"int32"))})}(d(y.squeeze(),r),b),heatmapScores:f,offsets:l,displacementFwd:p,displacementBwd:m}}),m=p.partSegmentation,g=p.heatmapScores,v=p.offsets,w=p.displacementFwd,y=p.displacementBwd;return c.dispose(),{partSegmentation:m,heatmapScores:g,offsets:v,displacementFwd:w,displacementBwd:y,padding:h,internalResolutionHeightAndWidth:f}},t.prototype.segmentPersonParts=function(t,e){return void 0===e&&(e=dt),a(this,void 0,void 0,function(){var n,r,o,a,u,d,f,l,c,h,p,m,g,v,w,y,b,x;return s(this,function(s){switch(s.label){case 0:return lt(e=i({},dt,e)),n=this.segmentPersonPartsActivation(t,e.internalResolution,e.segmentationThreshold),r=n.partSegmentation,o=n.heatmapScores,a=n.offsets,u=n.displacementFwd,d=n.displacementBwd,f=n.padding,l=n.internalResolutionHeightAndWidth,c=r.shape,h=c[0],p=c[1],[4,r.data()];case 1:return m=s.sent(),r.dispose(),[4,et([o,a,u,d])];case 2:return g=s.sent(),v=g[0],w=g[1],y=g[2],b=g[3],x=rt(x=C(v,w,y,b,this.baseModel.outputStride,e.maxDetections,e.scoreThreshold,e.nmsRadius),[h,p],l,f,!1),o.dispose(),a.dispose(),u.dispose(),d.dispose(),[2,{height:h,width:p,data:m,allPoses:x}]}})})},t.prototype.segmentMultiPersonParts=function(t,n){return void 0===n&&(n=ft),a(this,void 0,void 0,function(){var r,o,a,f,l,c,h,p,m,g,v,w,y,b,x,S,M,k,_,E,O,I,A=this;return s(this,function(s){switch(s.label){case 0:return ct(n=i({},ft,n)),r=Q(t),o=r[0],a=r[1],f=Z(n.internalResolution,this.baseModel.outputStride,[o,a]),l=tt(t,f),c=l.resized,h=l.padding,p=e.tidy(function(){var t=A.predictForMultiPersonInstanceSegmentationAndPart(c),r=t.segmentLogits,i=t.longOffsets,s=t.heatmapScores,l=t.offsets,p=t.displacementFwd,m=t.displacementBwd,g=t.partHeatmaps,v=$(r,[o,a],f,[[h.top,h.bottom],[h.left,h.right]],!0),w=$(g,[o,a],f,[[h.top,h.bottom],[h.left,h.right]],!0),y=i;return{segmentation:d(v.squeeze(),n.segmentationThreshold),longOffsets:y,heatmapScoresRaw:s,offsetsRaw:l,displacementFwdRaw:p,displacementBwdRaw:m,partSegmentation:function(t){var n=t.shape,r=n[0],o=n[1],i=n[2];return e.tidy(function(){var n=u(t),a=e.range(0,i,1,"int32").expandDims(1);return n.matMul(a).toInt().reshape([r,o])})}(w)}}),m=p.segmentation,g=p.longOffsets,v=p.heatmapScoresRaw,w=p.offsetsRaw,y=p.displacementFwdRaw,b=p.displacementBwdRaw,x=p.partSegmentation,[4,et([v,w,y,b])];case 1:return S=s.sent(),M=S[0],k=S[1],_=S[2],E=S[3],O=rt(O=C(M,k,_,E,this.baseModel.outputStride,n.maxDetections,n.scoreThreshold,n.nmsRadius),[o,a],f,h,!1),[4,P(m,g,x,O,o,a,this.baseModel.outputStride,f,h,n.scoreThreshold,n.refineSteps,n.minKeypointScore,n.maxDetections)];case 2:return I=s.sent(),c.dispose(),m.dispose(),g.dispose(),v.dispose(),w.dispose(),y.dispose(),b.dispose(),x.dispose(),[2,I]}})})},t.prototype.dispose=function(){this.baseModel.dispose()},t}();function pt(t){return a(this,void 0,void 0,function(){var r,o,i,a,u,d;return s(this,function(s){switch(s.label){case 0:if(r=t.outputStride,o=t.quantBytes,i=t.multiplier,null==e)throw new Error("Cannot find TensorFlow.js. If you are using a