(Enhance:Module:Request) 增加Chunked传输支持

modules/request.js

@@ -12,6 +12,7 @@ const fs = require('fs'),
   CONF = require('./config'),
   superagent = require('superagent'),
   superagentProxy = require('superagent-proxy');
+const { Readable } = require("stream");
 
 let logger;
 // 请求UA
@@ -109,6 +110,51 @@ class Request {
     }
     // 自定义body
     const _postData = Object.assign({}, opts.body, opts.data);
+    if (opts['useChunk'] == 1){
+      logger.debug("request with Chunked");
+      let _postarr = [];
+      for(var key in _postData){
+        if(_postData.hasOwnProperty(key)){
+          _postarr.push(`${key}=${encodeURIComponent(_postData[key])}`);
+        }
+      }
+      let antstream = new AntRead(_postarr.join("&"), {'step': parseInt(opts['chunkStepMin']), 'stepmax': parseInt(opts['chunkStepMax'])});
+      let _datasuccess = false; // 表示是否是 404 类shell
+      _request
+        .proxy(APROXY_CONF['uri'])
+        .type('form')
+        // .set('Content-Type', 'application/x-www-form-urlencoded')
+        .timeout(opts.timeout || REQ_TIMEOUT)
+        .ignoreHTTPS(opts['ignoreHTTPS'])
+        .parse((res, callback) => {
+          this.parse(opts['tag_s'], opts['tag_e'], (chunk) => {
+            event.sender.send('request-chunk-' + opts['hash'], chunk);
+          }, res, (err, ret)=>{
+            let buff = ret ? ret : new Buffer();
+            // 自动猜测编码
+            let encoding = detectEncoding(buff, {defaultEncoding: "unknown"});
+            logger.debug("detect encoding:", encoding);
+            encoding = encoding != "unknown" ? encoding : opts['encode'];
+            let text = iconv.decode(buff, encoding);
+            if (err && text == "") {
+              return event.sender.send('request-error-' + opts['hash'], err);
+            };
+            // 回调数据
+            event.sender.send('request-' + opts['hash'], {
+              text: text,
+              buff: buff,
+              encoding: encoding
+            });
+            _datasuccess = true;
+            callback(null, ret);
+          });
+        }).on('error', (err) => {
+          if(_datasuccess == false) {
+            return event.sender.send('request-error-' + opts['hash'], err);
+          }
+        });
+        antstream.pipe(_request);
+    }else{
       // 通过替换函数方式来实现发包方式切换, 后续可改成别的
       const old_send = _request.send;
       if(opts['useMultipart'] == 1) {
@@ -153,6 +199,7 @@ class Request {
           });
         });
       }
+  }
 
   /**
    * 监听下载请求
@@ -181,6 +228,49 @@ class Request {
     }
     // 自定义body
     const _postData = Object.assign({}, opts.body, opts.data);
+    if (opts['useChunk'] == 1){
+      logger.debug("request with Chunked");
+      let _postarr = [];
+      for(var key in _postData){
+        if(_postData.hasOwnProperty(key)){
+          _postarr.push(`${key}=${_postData[key]}`);
+        }
+      }
+      let antstream = new AntRead(_postarr.join("&"), {'step': parseInt(opts['chunkStepMin']), 'stepmax': parseInt(opts['chunkStepMax'])});
+      let _datasuccess = false; // 表示是否是 404 类shell
+      _request
+        .proxy(APROXY_CONF['uri'])
+        .type('form')
+        .ignoreHTTPS(opts['ignoreHTTPS'])
+        .pipe(through(
+          (chunk) => {
+            // 判断数据流中是否包含后截断符？长度++
+            let temp = chunk.indexOf(opts['tag_e']);
+            if (temp !== -1) {
+              indexEnd = Buffer.concat(tempData).length + temp;
+            };
+            tempData.push(chunk);
+            event.sender.send('download-progress-' + opts['hash'], chunk.length);
+          },
+          () => {
+            let tempDataBuffer = Buffer.concat(tempData);
+
+            indexStart = tempDataBuffer.indexOf(opts['tag_s']) || 0;
+            // 截取最后的数据
+            let finalData = new Buffer(tempDataBuffer.slice(
+              indexStart + opts['tag_s'].length,
+              indexEnd
+            ), 'binary');
+            // 写入文件流&&关闭
+            rs.write(finalData);
+            rs.close();
+            event.sender.send('download-' + opts['hash'], finalData.length);
+            // 删除内存数据
+            finalData = tempDataBuffer = tempData = null;
+          }
+      ));
+      antstream.pipe(_request);
+    }else{
       // 通过替换函数方式来实现发包方式切换, 后续可改成别的
       const old_send = _request.send;
       if(opts['useMultipart'] == 1) {
@@ -224,6 +314,7 @@ class Request {
           }
       ));
     }
+  }
 
   /**
    * 二进制数据流解析
@@ -326,4 +417,59 @@ function detectEncoding(buffer, options) {
   }
 };
 
+
+/**
+ * 控步长的可读流
+ * @param data  [string|buffer] 输入源
+ * @param options {} 配置
+ *    step     步长
+ *    stepmax  最大步长,默认与步长相等,如果大于步长，则每次读取时后随机返回 [step, stepmax] 长度的数据
+ */
+class AntRead extends Readable {
+  constructor(data, options={}) {
+    super();
+    this.index = 0;
+    let o = {};
+    o.step = options.hasOwnProperty('step') ? parseInt(options['step']) : 2;
+    o.step = o.step < 1 ? 2 : o.step;
+    o.stepmax = options.hasOwnProperty('stepmax') ? options['stepmax'] : o.step ;
+    if (o.stepmax < o.step) {
+      o.stepmax = o.step;
+    }
+    let chunk;
+    if('string' === typeof data) {
+      chunk = data;
+    }else if('object' === typeof data && Buffer.isBuffer(data)) { // buffer
+      chunk = new Buffer(data).toString();
+    }else{
+      throw Error("data must be string, buffer.");
+    }
+    this.chunk = chunk;
+    this.o = o;
+  }
+
+  // 重写自定义的可读流的 _read 方法
+  _read() {
+      let step = this.randomNum(this.o.step, this.o.stepmax);
+      if (this.index >= this.chunk.length) {
+          this.push(null);
+      }else{
+          this.push(this.chunk.substring(this.index, this.index + step) + "");
+      }
+      this.index += step;
+  }
+
+  // random [n, m]
+  randomNum(minNum, maxNum){ 
+    switch(arguments.length){ 
+        case 1: 
+          return parseInt(Math.random()*minNum+1,10);
+        case 2: 
+          return parseInt(Math.random()*(maxNum-minNum+1)+minNum,10); 
+        default:
+          return 0; 
+    } 
+  }
+}
+
 module.exports = Request;

source/core/base.js

@@ -245,6 +245,9 @@ class Base {
           tag_e: opt['tag_e'],
           encode: this.__opts__['encode'],
           ignoreHTTPS: (this.__opts__['otherConf'] || {})['ignore-https'] === 1,
+          useChunk: (this.__opts__['otherConf'] || {})['use-chunk'] === 1,
+          chunkStepMin: (this.__opts__['otherConf'] || {})['chunk-step-byte-min'] || 2,
+          chunkStepMax: (this.__opts__['otherConf'] || {})['chunk-step-byte-max'] || 3,
           useMultipart: (this.__opts__['otherConf'] || {})['use-multipart'] === 1,
           timeout: parseInt((this.__opts__['otherConf'] || {})['request-timeout']),
           headers: (this.__opts__['httpConf'] || {})['headers'] || {},

source/language/en.js

@@ -169,6 +169,13 @@ module.exports = {
       otherConf: {
         nohttps: 'Ignore HTTPS certificate',
         usemultipart: 'Use Multipart send payload',
+        chunk: {
+          title: 'Chunked Transfer (Experimentally)',
+          usechunk: 'Use Chunked send payload.',
+          min: 'Min Block',
+          max: 'Max Block',
+          exphint: 'This feature is currently experimental and cannot be used with Multipart. Some types of servers may not support Chunked transfers. In addition, it is recommended to set the timeout period to 30s or more to avoid data transmission when the network speed is not good.',
+        },
         terminalCache: "Use the terminal's cache",
         filemanagerCache: "Use the filemanager's cache",
         uploadFragment: "Upload File Fragmentation Size",

source/language/zh.js

@@ -170,6 +170,13 @@ module.exports = {
       otherConf: {
         nohttps: '忽略HTTPS证书',
         usemultipart: '使用 Multipart 发包',
+        chunk: {
+          title: '分块传输(实验性功能)',
+          usechunk: '开启分块传输发包',
+          min: '最小分块',
+          max: '最大分块',
+          exphint: '该功能目前为实验性功能, 无法与 Multipart 同时使用,部分类型的服务端可能不支持Chunked传输。此外,建议超时时长设置30s以上,避免网速不好的情况下影响数据传输。',
+        },
         terminalCache: '虚拟终端使用缓存',
         filemanagerCache: '文件管理使用缓存',
         uploadFragment: '上传文件分片大小',

source/modules/shellmanager/list/form.js

@@ -318,6 +318,9 @@ class Form {
     const opt = Object.assign({}, {
       'ignore-https': 0,
       'use-multipart': 0,
+      'use-chunk': 0,
+      'chunk-step-byte-min': 2,
+      'chunk-step-byte-max': 3,
       'terminal-cache': 0,
       'filemanager-cache': 1,
       'upload-fragment': '500',
@@ -334,7 +337,60 @@ class Form {
         }, {
           type: "checkbox", name: 'use-multipart', label: LANG['list']['otherConf']['usemultipart'],
           checked: opt['use-multipart'] === 1
-        }, {
+        }, { type: 'fieldset', offsetLeft: 0, label: LANG['list']['otherConf']['chunk']['title'], list: [
+          { type: 'block', offsetLeft: 0, list: [
+            {
+              type: "checkbox", name: 'use-chunk', label: LANG['list']['otherConf']['chunk']['usechunk'], checked: opt['use-chunk'] === 1
+            },
+          ]},
+          { type: 'block', offsetLeft: 0, list: [
+            { type:'label', label: LANG['list']['otherConf']['chunk']['min']},
+            { type:'newcolumn' },
+            {
+              type: 'combo', label: '/byte', validate: 'ValidNumeric', inputWidth: 50, name: "chunk-step-byte-min",
+              options: ((items) => {
+                let ret = [];
+                // 如果自定义的路径不在items里，则++
+                if (items.indexOf(opt['chunk-step-byte-min']) === -1) {
+                  items.unshift(opt['chunk-step-byte-min']);
+                }
+                items.map((_) => {
+                  ret.push({
+                    text: _,
+                    value: _,
+                    selected: opt['chunk-step-byte-min'] === _
+                  })
+                });
+                return ret;
+              })([
+                '2', '4', '10', '50', '100', '500'
+              ])
+            },
+            { type:'newcolumn',},
+            { type:'label', label: LANG['list']['otherConf']['chunk']['max'], offsetLeft: 30,},
+            { type:'newcolumn' },
+            {
+              type: 'combo', label: '/byte', validate: 'ValidNumeric', inputWidth: 50, name: "chunk-step-byte-max",
+              options: ((items) => {
+                let ret = [];
+                // 如果自定义的路径不在items里，则++
+                if (items.indexOf(opt['chunk-step-byte-max']) === -1) {
+                  items.unshift(opt['chunk-step-byte-max']);
+                }
+                items.map((_) => {
+                  ret.push({
+                    text: _,
+                    value: _,
+                    selected: opt['chunk-step-byte-max'] === _
+                  })
+                });
+                return ret;
+              })([
+                '2', '4', '10', '50', '100', '500'
+              ])
+            },
+          ]},
+        ]}, {
           type: "checkbox", name: 'terminal-cache', label: LANG['list']['otherConf']['terminalCache'],
           checked: opt['terminal-cache'] === 1
         }, {
@@ -405,6 +461,28 @@ class Form {
           ])
         }
       ]}], true);
+    form.attachEvent('onChange', (name, value, state)=>{
+      switch(name){
+        case 'use-multipart':
+          if(state == true && form.isItemChecked('use-chunk')) {
+            form.uncheckItem('use-chunk');
+          }
+        break;
+        case 'use-chunk':
+          if(state == true && form.isItemChecked('use-multipart')) {
+            form.uncheckItem('use-multipart');
+          }
+          if(state == true) {
+            layer.open({
+              title: LANG_T['info']
+              ,content: LANG['list']['otherConf']['chunk']['exphint']
+            });            
+          }
+        break;
+        default:
+        break;
+      }
+    });
     return form;
   }