nodejs后台babel在线热编译jsx

浏览器加载react/vue组件时，遇到es6转es5，jsx转js...时，一种方法是用webpack离线编译，一种方法是在后台用babel在线热编译（为了效率部署前可以预热）。

我比较喜欢在线热编译，好处是发布时快，不经过build直接源码发布，并可以避免忘记编译步骤导致bug。

为了提供效率，把热编译结果保存为文件缓存起来。先检查是否有编译后缓存文件且没有过期，有就直接读取，否者编译后再读取。

node.js代码。

let fs=require('fs');
let babel = require("@babel/core");
let babel_preset_env=require('@babel/preset-env');
let babel_preset_react=require("@babel/preset-react");

function transform(f,cb){
  let fc=f+'.js';
  function _trans(){
    fs.readFile(f,'utf8',function(err,code){
      if (err){cb(err)}
      else{
        let r=babel.transformSync(code,{presets:[babel_preset_env,babel_preset_react]});
        code=r.code;
        cb(null,code);
        fs.writeFile(fc,code,function(err){
        });
      }
    });
  }
  fs.lstat(fc,function(err,fcStat){
    if (err){
      _trans();
    }
    else{
      fs.lstat(f,function(err,fStat){
        if (err) cb(err);
        else{
          if (fcStat.birthtimeMs<fStat.mtimeMs){
            _trans();
          }
          else{
            fs.readFile(f,'utf8',function(err,code){
              cb(err,code);
            });
          }
        }
      });
    }
  });
}
transform('s.jsx');

但在多并发时，问题来了：多个并发任务可能会同时都检查到缓存不存在，然后开始编译.......很浪费，其实只需要一个任务来编译，其它任务等待编译结束后再读取缓存。

其实有点复杂，涉及到文件锁机制，阿里“通义千问”建议用proper-lockfile，我没用。模拟一下：

/**
 * 测试多进程下，判断一个文件是否存在，不存在才生成内容创建文件
 * 难点：
 * （1）避免多个进程在生成文件内容。
 * （2）一个进程如何等待正在生成文件的进程生成完成再读取。
 * （3）等待的效率，降低cpu占用
 */
let path=require('path');
let fs=require('fs');
let fn=path.join(__dirname,'a.txt');

function rw5(task){
  let fnlock=fn+'.lock';
  //递归获取锁
  function lock(cb){
    fs.open(fnlock,'wx',function(err,fhandle){
      if (err){
        console.log(task,' locked,try again...');
        setTimeout(function(){
          lock(cb);
        },1);
      }
      else{
        fs.close(fhandle,function(err){
          if (err) console.log(err);
          console.log(task,' got lock');
          cb();
        });
      }
    });
  }
  function unlock(){
    fs.unlink(fnlock,function(err){if (err) console.log(err)});
  }
  function read(cb){
    fs.readFile(fn,'utf8',function(err,data){
      //读不到正在写入的内容
      if (err) console.log(err);
      else console.log(task,' readed:',data);
      if (cb) cb();
    });
  }
  function write(cb){
    let content='hello';
    //用setTimeout模拟一个长时间的content计算过程，比如babel转码
    setTimeout(function(){
      fs.writeFile(fn,content,function(err,data){
        console.log(task,' writed:',content);
        if (cb) cb();
      });
    }, 1000);
  }
  fs.access(fn,function(err,data){
    if (err){
      console.log(task,' not exists');
      lock(function(){
        fs.access(fn,function(err,data){
          if (err){
            console.log(task,' not exists');
            write(function(){
              unlock();
            });
          }
          else{
            read(function(){
              unlock();
            });
          }
        });
      });
    }
    else{
      read();
    }
  });
}
rw5(1);
rw5(2);

看到的运行记录，可能如下：

D:\work\Source\yujiang.Foil.Node\test\filerw>node multirw.js
1  not exists
2  not exists
1  got lock
2  locked,try again...
1  not exists
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
2  locked,try again...
1  writed: hello
2  locked,try again...
2  got lock
2  readed: hello

封装一下：

yjFile.js

let fs=require('fs');
let path=require('path');
let debug=[console.log,function(){}][0];
/**
 * 热转换文件，有缓存文件且没有过期时，直接读取，否则转换后保存读取。
 * 考虑了异步并发的状况，转化动作只会有一个任务执行，其它等待。
 * @param {*} ops 
 * @param {string} ops.fSource 源文件
 * @param {string} ops.fTarget 转码后的文件
 * @param {int} ops.tryLockInterval 尝试锁定文件的时间间隔，单位：毫秒，预设值：1。数字越大，cpu占用越少，但任务时间拖得越长。
 * @param {any} ops.taskID debug时日志显示的函数调用任务ID
 */
function hotTransform(ops){
  if (!ops) ops={};
  if (ops.tryLockInterval==undefined) ops.tryLockInterval=1;
  let fLock=ops.fSource+'.lock';
  //递归获取锁
  function lock(cb){
    debug(ops.taskID,'try lock...');
    fs.open(fLock,'wx',function(err,fHandle){
      if (err){
        setTimeout(function(){
          lock(cb);
        },ops.tryLockInterval);
      }
      else{
        fs.close(fHandle,function(err){
          if (err) debug(ops.taskID,err);
          debug(ops.taskID,'......got lock');
          cb();
        });
      }
    });
  }
  function unlock(){
    fs.unlink(fLock,function(err){if (err) debug(ops.taskID,err)});
    debug(ops.taskID,'unlock');
  }
  function read(cb){
    fs.readFile(ops.fTarget,'utf8',function(err,data){
      if (err){
        debug(ops.taskID,err);
      }
      else{
        debug(ops.taskID,'readed:',data);
      }
      if (cb) cb();
      ops.cb(err,data);
    });
  }
  function write(data,cb){
    fs.writeFile(ops.fTarget,data,function(err,data2){
      if (err){
        if (err.code=='ENOENT'){
          let dir=path.dirname(ops.fTarget);
          fs.mkdir(dir,function(err){
            if (err){
              if (cb) cb(err);
            }
            else{
              write(data,cb);
            }
          });
        }
        else{
          if (cb) cb();
          debug(ops.taskID,err);
        }
      }
      else{
        if (cb) cb();
        debug(ops.taskID,'========= writed:',data);
      }
    });
  }
  function trans(){
    function handleError(e){
      unlock();
      debug(ops.taskID,e);
      ops.cb(e);
    }
    function _t(){
      ops.transform(function(err,data){
        if (err){
          handleError(err);
        }
        else{
          try{
            write(data,function(err){
              unlock();
              //注意：ops.cb要放在这里，如果放在write外面，可能ops.cb通知任务结束，子进程马上被杀掉，unlock不会被执行到。
              ops.cb(err,data);
            });
          }
          catch(e){
            handleError(e);
          }
        }
      });
    }
    lock(function(){
      //要加锁后，才能读写ops.fTarget文件。要保证能解锁。
      try{
        fs.lstat(ops.fTarget,function(err,targetStat){
          try{
            if (err){
              debug(ops.taskID,'locked,not exists');
              _t();
            }
            else{
              fs.lstat(ops.fSource,function(err,sourceStat){
                if (err) handleError(err);
                else{
                  if (targetStat.birthtimeMs<sourceStat.mtimeMs){
                    debug(ops.taskID,'locked,expired');
                    _t();
                  }
                  else read(function(){
                    unlock();
                  });
                }
              });
            }
          }
          catch(e){
            handleError(e);
          }
        });
      }
      catch(e){
        handleError(e);
      }
    });
  }
  //为了效率：先判断缓存文件是否可用，可用直接用，否则要进入trans转码（转码锁文件效率很低）
  fs.lstat(ops.fTarget,function(err,targetStat){
    if (err){
      debug(ops.taskID,'not exists');
      trans();
    }
    else{
      fs.lstat(ops.fSource,function(err,sourceStat){
        //并发时，这时可能ops.fTarget已经更新了，无所谓，只是进入trans再加锁读取。
        if (err){
          ops.cb(err);
        }
        else{
          if (targetStat.birthtimeMs<sourceStat.mtimeMs){
            trans();
          }
          else read();
        }
      });
    }
  });
}
module.exports.hotTransform=hotTransform;

测试一下：

testCase_hotTransform.js

let path=require('path');
let fs=require('fs');
let yjFile=require('../../src/yjFile.js');

const CNST_taskCount=5;
let endTaskCount=0;
let fSource=path.join(__dirname,'s.txt');
let ops={
  fSource,
  fTarget:path.join(__dirname,'dist/t.txt'),
  transform(cb){
    fs.readFile(fSource,'utf8',function(err,data){
      data=data+',tranformed.';
      cb(err,data);
    });
  },
  cb(err,data){
    let t2=new Date().getTime();
    console.log('-------',this.taskID,t2-this.startTime+'ms',err,data);
    endTaskCount++;
    if (endTaskCount==CNST_taskCount){
      if (process.send){
        process.send({cmd:'end'});
      }
    }
  }
}

function start(taskID,waitTime){
  //增加延迟，让子进程的任务和主进程的任务尽量能同时并发执行，增加并发碰撞机会。
  //至少主进程内的2个任务是并发的，子进程内的2个任务是并发的。
  setTimeout(function(){
    for(let i=0;i<CNST_taskCount;i++){
      let ops1=Object.assign({taskID:taskID+i},ops);
      ops1.startTime=new Date().getTime();
      //console.log(ops1);
      yjFile.hotTransform(ops1);
    }
  },waitTime);
}

process.on("message",function({cmd,data}){
  switch (cmd){
    case 'start':
      start(data.taskID,0);
      break;
  }
});

module.exports.start=start;
module.exports.taskCount=CNST_taskCount;

run.js

let path=require('path');
let {fork} = require('child_process');
let testProcess = fork(path.join(__dirname,'testCase_hotTransform.js'));
testProcess.send({cmd:'start',data:{taskID:1}},function(err,data){
  let test=require('./testCase_hotTransform.js');
  //增加延迟，让子进程的任务和主进程的任务大概能“同时”进行，才能真正模拟并发。
  //否则，总是主进程发起的任务先执行完。
  test.start(test.taskCount+1,100);
});
testProcess.on("message",function({cmd,data}){
  switch (cmd){
    case 'end':
      testProcess.kill();
      break;
  }
});

主进程开启5个并发，并fork一个子进程，子进程里面也开启5个并发。

先随便建立一个s.txt文件，node run.js执行，可能看到的结果：

D:\work\Source\yujiang.Foil.Node\test\filerw>node run.js
6 not exists
6 try lock...
7 not exists
7 try lock...
8 not exists
8 try lock...
9 not exists
9 try lock...
10 not exists
10 try lock...
6 ......got lock
7 try lock...
8 try lock...
9 try lock...
10 try lock...
6 locked,not exists
1 not exists
7 try lock...
8 try lock...
9 try lock...
1 try lock...
10 try lock...
2 not exists
2 try lock...
7 try lock...
8 try lock...
9 try lock...
3 not exists
3 try lock...
4 not exists
10 try lock...
4 try lock...
7 try lock...
5 not exists
8 try lock...
5 try lock...
9 try lock...
1 try lock...
2 try lock...
10 try lock...
7 try lock...
3 try lock...
8 try lock...
4 try lock...
9 try lock...
6 unlock
5 try lock...
------- 6 56ms undefined source,tranformed.
6 ========= writed: source,tranformed.
1 try lock...
10 try lock...
7 try lock...
2 try lock...
3 try lock...
4 try lock...
5 ......got lock
8 try lock...
1 try lock...
9 try lock...
2 try lock...
10 try lock...
3 try lock...
7 try lock...
4 try lock...
1 try lock...
2 try lock...
3 try lock...
4 try lock...
5 readed: source,tranformed.
5 unlock
------- 5 52ms null source,tranformed.
1 try lock...
8 try lock...
2 try lock...
3 try lock...
9 try lock...
4 try lock...
10 try lock...
2 try lock...
7 try lock...
1 ......got lock
3 try lock...
4 try lock...
1 readed: source,tranformed.
1 unlock
------- 1 72ms null source,tranformed.
2 try lock...
3 try lock...
4 try lock...
8 try lock...
9 try lock...
10 try lock...
7 try lock...
3 try lock...
2 ......got lock
8 try lock...
9 try lock...
4 try lock...
10 try lock...
3 try lock...
7 try lock...
4 try lock...
8 try lock...
3 try lock...
9 try lock...
10 try lock...
2 readed: source,tranformed.
7 try lock...
2 unlock
------- 2 83ms null source,tranformed.
4 try lock...
3 try lock...
4 ......got lock
8 try lock...
9 try lock...
10 try lock...
3 try lock...
7 try lock...
8 try lock...
4 readed: source,tranformed.
4 unlock
------- 4 93ms null source,tranformed.
3 try lock...
3 ......got lock
3 readed: source,tranformed.
3 unlock
------- 3 98ms null source,tranformed.
9 try lock...
10 try lock...
7 try lock...
8 try lock...
9 ......got lock
10 try lock...
7 try lock...
8 try lock...
10 try lock...
7 try lock...
8 try lock...
9 readed: source,tranformed.
9 unlock
------- 9 131ms null source,tranformed.
10 try lock...
7 try lock...
8 try lock...
10 ......got lock
7 try lock...
8 try lock...
7 try lock...
10 readed: source,tranformed.
10 unlock
------- 10 142ms null source,tranformed.
8 try lock...
7 try lock...
8 ......got lock
7 try lock...
8 readed: source,tranformed.
8 unlock
------- 8 148ms null source,tranformed.
7 try lock...
7 ......got lock
7 readed: source,tranformed.
7 unlock
------- 7 157ms null source,tranformed.

从结果看，10个并发任务，各自消耗的时间为：

------- 6 56ms
------- 5 52ms
------- 1 72ms
------- 2 83ms
------- 4 93ms
------- 3 98ms
------- 9 131ms
------- 10 142ms
------- 8 148ms
------- 7 157ms

第一个完成的任务是6号，会面等待的任务，本应该马上获得数据结束，但是花了太多的时间在try lock上。如何再提高效率？