Skip to content

A
antSword
Commit 354eed16 authored by Medicean's avatar Medicean
Browse files
Options

(Enhance:Module:Request) 增加Chunked传输支持

parent 03c6d6f4
Hide whitespace changes
Inline Side-by-side
Showing with 323 additions and 82 deletions
modules/request.js
View file @ 354eed16
...@@ -12,6 +12,7 @@ const fs = require('fs'), ...@@ -12,6 +12,7 @@ const fs = require('fs'),
CONF = require('./config'), CONF = require('./config'),
superagent = require('superagent'), superagent = require('superagent'),
superagentProxy = require('superagent-proxy'); superagentProxy = require('superagent-proxy');
const { Readable } = require("stream");
let logger; let logger;
// 请求UA // 请求UA
...@@ -109,49 +110,95 @@ class Request { ...@@ -109,49 +110,95 @@ class Request {
} }
// 自定义body // 自定义body
const _postData = Object.assign({}, opts.body, opts.data); const _postData = Object.assign({}, opts.body, opts.data);
// 通过替换函数方式来实现发包方式切换, 后续可改成别的 if (opts['useChunk'] == 1){
const old_send = _request.send; logger.debug("request with Chunked");
if(opts['useMultipart'] == 1) { let _postarr = [];
_request.send = _request.field; for(var key in _postData){
}else{ if(_postData.hasOwnProperty(key)){
_request.send = old_send; _postarr.push(`${key}=${encodeURIComponent(_postData[key])}`);
}
_request
.proxy(APROXY_CONF['uri'])
.type('form')
// 超时
.timeout(opts.timeout || REQ_TIMEOUT)
// 忽略HTTPS
.ignoreHTTPS(opts['ignoreHTTPS'])
.send(_postData)
.parse((res, callback) => {
this.parse(opts['tag_s'], opts['tag_e'], (chunk) => {
event.sender.send('request-chunk-' + opts['hash'], chunk);
}, res, callback);
})
.end((err, ret) => {
if (!ret) {
// 请求失败 TIMEOUT
return event.sender.send('request-error-' + opts['hash'], err);
} }
let buff = ret.hasOwnProperty('body') ? ret.body : new Buffer(); }
// 解码 let antstream = new AntRead(_postarr.join("&"), {'step': parseInt(opts['chunkStepMin']), 'stepmax': parseInt(opts['chunkStepMax'])});
let text = ""; let _datasuccess = false; // 表示是否是 404 类shell
// 自动猜测编码 _request
let encoding = detectEncoding(buff, {defaultEncoding:"unknown"}); .proxy(APROXY_CONF['uri'])
logger.debug("detect encoding:", encoding); .type('form')
encoding = encoding != "unknown" ? encoding : opts['encode']; // .set('Content-Type', 'application/x-www-form-urlencoded')
text = iconv.decode(buff, encoding); .timeout(opts.timeout || REQ_TIMEOUT)
if (err && text == "") { .ignoreHTTPS(opts['ignoreHTTPS'])
return event.sender.send('request-error-' + opts['hash'], err); .parse((res, callback) => {
}; this.parse(opts['tag_s'], opts['tag_e'], (chunk) => {
// 回调数据 event.sender.send('request-chunk-' + opts['hash'], chunk);
event.sender.send('request-' + opts['hash'], { }, res, (err, ret)=>{
text: text, let buff = ret ? ret : new Buffer();
buff: buff, // 自动猜测编码
encoding: encoding let encoding = detectEncoding(buff, {defaultEncoding: "unknown"});
logger.debug("detect encoding:", encoding);
encoding = encoding != "unknown" ? encoding : opts['encode'];
let text = iconv.decode(buff, encoding);
if (err && text == "") {
return event.sender.send('request-error-' + opts['hash'], err);
};
// 回调数据
event.sender.send('request-' + opts['hash'], {
text: text,
buff: buff,
encoding: encoding
});
_datasuccess = true;
callback(null, ret);
});
}).on('error', (err) => {
if(_datasuccess == false) {
return event.sender.send('request-error-' + opts['hash'], err);
}
}); });
}); antstream.pipe(_request);
}else{
// 通过替换函数方式来实现发包方式切换, 后续可改成别的
const old_send = _request.send;
if(opts['useMultipart'] == 1) {
_request.send = _request.field;
}else{
_request.send = old_send;
}
_request
.proxy(APROXY_CONF['uri'])
.type('form')
// 超时
.timeout(opts.timeout || REQ_TIMEOUT)
// 忽略HTTPS
.ignoreHTTPS(opts['ignoreHTTPS'])
.send(_postData)
.parse((res, callback) => {
this.parse(opts['tag_s'], opts['tag_e'], (chunk) => {
event.sender.send('request-chunk-' + opts['hash'], chunk);
}, res, callback);
})
.end((err, ret) => {
if (!ret) {
// 请求失败 TIMEOUT
return event.sender.send('request-error-' + opts['hash'], err);
}
let buff = ret.hasOwnProperty('body') ? ret.body : new Buffer();
// 解码
let text = "";
// 自动猜测编码
let encoding = detectEncoding(buff, {defaultEncoding:"unknown"});
logger.debug("detect encoding:", encoding);
encoding = encoding != "unknown" ? encoding : opts['encode'];
text = iconv.decode(buff, encoding);
if (err && text == "") {
return event.sender.send('request-error-' + opts['hash'], err);
};
// 回调数据
event.sender.send('request-' + opts['hash'], {
text: text,
buff: buff,
encoding: encoding
});
});
}
} }
/** /**
...@@ -181,48 +228,92 @@ class Request { ...@@ -181,48 +228,92 @@ class Request {
} }
// 自定义body // 自定义body
const _postData = Object.assign({}, opts.body, opts.data); const _postData = Object.assign({}, opts.body, opts.data);
// 通过替换函数方式来实现发包方式切换, 后续可改成别的 if (opts['useChunk'] == 1){
const old_send = _request.send; logger.debug("request with Chunked");
if(opts['useMultipart'] == 1) { let _postarr = [];
_request.send = _request.field; for(var key in _postData){
}else{ if(_postData.hasOwnProperty(key)){
_request.send = old_send; _postarr.push(`${key}=${_postData[key]}`);
}
_request
.proxy(APROXY_CONF['uri'])
.type('form')
// 设置超时会导致文件过大时写入出错
// .timeout(timeout)
// 忽略HTTPS
.ignoreHTTPS(opts['ignoreHTTPS'])
.send(_postData)
.pipe(through(
(chunk) => {
// 判断数据流中是否包含后截断符？长度++
let temp = chunk.indexOf(opts['tag_e']);
if (temp !== -1) {
indexEnd = Buffer.concat(tempData).length + temp;
};
tempData.push(chunk);
event.sender.send('download-progress-' + opts['hash'], chunk.length);
},
() => {
let tempDataBuffer = Buffer.concat(tempData);
indexStart = tempDataBuffer.indexOf(opts['tag_s']) || 0;
// 截取最后的数据
let finalData = new Buffer(tempDataBuffer.slice(
indexStart + opts['tag_s'].length,
indexEnd
), 'binary');
// 写入文件流&&关闭
rs.write(finalData);
rs.close();
event.sender.send('download-' + opts['hash'], finalData.length);
// 删除内存数据
finalData = tempDataBuffer = tempData = null;
} }
}
let antstream = new AntRead(_postarr.join("&"), {'step': parseInt(opts['chunkStepMin']), 'stepmax': parseInt(opts['chunkStepMax'])});
let _datasuccess = false; // 表示是否是 404 类shell
_request
.proxy(APROXY_CONF['uri'])
.type('form')
.ignoreHTTPS(opts['ignoreHTTPS'])
.pipe(through(
(chunk) => {
// 判断数据流中是否包含后截断符？长度++
let temp = chunk.indexOf(opts['tag_e']);
if (temp !== -1) {
indexEnd = Buffer.concat(tempData).length + temp;
};
tempData.push(chunk);
event.sender.send('download-progress-' + opts['hash'], chunk.length);
},
() => {
let tempDataBuffer = Buffer.concat(tempData);
indexStart = tempDataBuffer.indexOf(opts['tag_s']) || 0;
// 截取最后的数据
let finalData = new Buffer(tempDataBuffer.slice(
indexStart + opts['tag_s'].length,
indexEnd
), 'binary');
// 写入文件流&&关闭
rs.write(finalData);
rs.close();
event.sender.send('download-' + opts['hash'], finalData.length);
// 删除内存数据
finalData = tempDataBuffer = tempData = null;
}
));
antstream.pipe(_request);
}else{
// 通过替换函数方式来实现发包方式切换, 后续可改成别的
const old_send = _request.send;
if(opts['useMultipart'] == 1) {
_request.send = _request.field;
}else{
_request.send = old_send;
}
_request
.proxy(APROXY_CONF['uri'])
.type('form')
// 设置超时会导致文件过大时写入出错
// .timeout(timeout)
// 忽略HTTPS
.ignoreHTTPS(opts['ignoreHTTPS'])
.send(_postData)
.pipe(through(
(chunk) => {
// 判断数据流中是否包含后截断符？长度++
let temp = chunk.indexOf(opts['tag_e']);
if (temp !== -1) {
indexEnd = Buffer.concat(tempData).length + temp;
};
tempData.push(chunk);
event.sender.send('download-progress-' + opts['hash'], chunk.length);
},
() => {
let tempDataBuffer = Buffer.concat(tempData);
indexStart = tempDataBuffer.indexOf(opts['tag_s']) || 0;
// 截取最后的数据
let finalData = new Buffer(tempDataBuffer.slice(
indexStart + opts['tag_s'].length,
indexEnd
), 'binary');
// 写入文件流&&关闭
rs.write(finalData);
rs.close();
event.sender.send('download-' + opts['hash'], finalData.length);
// 删除内存数据
finalData = tempDataBuffer = tempData = null;
}
)); ));
}
} }
/** /**
...@@ -326,4 +417,59 @@ function detectEncoding(buffer, options) { ...@@ -326,4 +417,59 @@ function detectEncoding(buffer, options) {
} }
}; };
/**
* 控步长的可读流
* @param data [string|buffer] 输入源
* @param options {} 配置
* step 步长
* stepmax 最大步长,默认与步长相等,如果大于步长，则每次读取时后随机返回 [step, stepmax] 长度的数据
*/
class AntRead extends Readable {
constructor(data, options={}) {
super();
this.index = 0;
let o = {};
o.step = options.hasOwnProperty('step') ? parseInt(options['step']) : 2;
o.step = o.step < 1 ? 2 : o.step;
o.stepmax = options.hasOwnProperty('stepmax') ? options['stepmax'] : o.step ;
if (o.stepmax < o.step) {
o.stepmax = o.step;
}
let chunk;
if('string' === typeof data) {
chunk = data;
}else if('object' === typeof data && Buffer.isBuffer(data)) { // buffer
chunk = new Buffer(data).toString();
}else{
throw Error("data must be string, buffer.");
}
this.chunk = chunk;
this.o = o;
}
// 重写自定义的可读流的 _read 方法
_read() {
let step = this.randomNum(this.o.step, this.o.stepmax);
if (this.index >= this.chunk.length) {
this.push(null);
}else{
this.push(this.chunk.substring(this.index, this.index + step) + "");
}
this.index += step;
}
// random [n, m]
randomNum(minNum, maxNum){
switch(arguments.length){
case 1:
return parseInt(Math.random()*minNum+1,10);
case 2:
return parseInt(Math.random()*(maxNum-minNum+1)+minNum,10);
default:
return 0;
}
}
}
module.exports = Request; module.exports = Request;
source/core/base.js
View file @ 354eed16
...@@ -245,6 +245,9 @@ class Base { ...@@ -245,6 +245,9 @@ class Base {
tag_e: opt['tag_e'], tag_e: opt['tag_e'],
encode: this.__opts__['encode'], encode: this.__opts__['encode'],
ignoreHTTPS: (this.__opts__['otherConf'] || {})['ignore-https'] === 1, ignoreHTTPS: (this.__opts__['otherConf'] || {})['ignore-https'] === 1,
useChunk: (this.__opts__['otherConf'] || {})['use-chunk'] === 1,
chunkStepMin: (this.__opts__['otherConf'] || {})['chunk-step-byte-min'] || 2,
chunkStepMax: (this.__opts__['otherConf'] || {})['chunk-step-byte-max'] || 3,
useMultipart: (this.__opts__['otherConf'] || {})['use-multipart'] === 1, useMultipart: (this.__opts__['otherConf'] || {})['use-multipart'] === 1,
timeout: parseInt((this.__opts__['otherConf'] || {})['request-timeout']), timeout: parseInt((this.__opts__['otherConf'] || {})['request-timeout']),
headers: (this.__opts__['httpConf'] || {})['headers'] || {}, headers: (this.__opts__['httpConf'] || {})['headers'] || {},
......
source/language/en.js
View file @ 354eed16
...@@ -169,6 +169,13 @@ module.exports = { ...@@ -169,6 +169,13 @@ module.exports = {
otherConf: { otherConf: {
nohttps: 'Ignore HTTPS certificate', nohttps: 'Ignore HTTPS certificate',
usemultipart: 'Use Multipart send payload', usemultipart: 'Use Multipart send payload',
chunk: {
title: 'Chunked Transfer (Experimentally)',
usechunk: 'Use Chunked send payload.',
min: 'Min Block',
max: 'Max Block',
exphint: 'This feature is currently experimental and cannot be used with Multipart. Some types of servers may not support Chunked transfers. In addition, it is recommended to set the timeout period to 30s or more to avoid data transmission when the network speed is not good.',
},
terminalCache: "Use the terminal's cache", terminalCache: "Use the terminal's cache",
filemanagerCache: "Use the filemanager's cache", filemanagerCache: "Use the filemanager's cache",
uploadFragment: "Upload File Fragmentation Size", uploadFragment: "Upload File Fragmentation Size",
......
source/language/zh.js
View file @ 354eed16
...@@ -170,6 +170,13 @@ module.exports = { ...@@ -170,6 +170,13 @@ module.exports = {
otherConf: { otherConf: {
nohttps: '忽略HTTPS证书', nohttps: '忽略HTTPS证书',
usemultipart: '使用 Multipart 发包', usemultipart: '使用 Multipart 发包',
chunk: {
title: '分块传输(实验性功能)',
usechunk: '开启分块传输发包',
min: '最小分块',
max: '最大分块',
exphint: '该功能目前为实验性功能, 无法与 Multipart 同时使用,部分类型的服务端可能不支持Chunked传输。此外,建议超时时长设置30s以上,避免网速不好的情况下影响数据传输。',
},
terminalCache: '虚拟终端使用缓存', terminalCache: '虚拟终端使用缓存',
filemanagerCache: '文件管理使用缓存', filemanagerCache: '文件管理使用缓存',
uploadFragment: '上传文件分片大小', uploadFragment: '上传文件分片大小',
......
source/modules/shellmanager/list/form.js
View file @ 354eed16
...@@ -318,6 +318,9 @@ class Form { ...@@ -318,6 +318,9 @@ class Form {
const opt = Object.assign({}, { const opt = Object.assign({}, {
'ignore-https': 0, 'ignore-https': 0,
'use-multipart': 0, 'use-multipart': 0,
'use-chunk': 0,
'chunk-step-byte-min': 2,
'chunk-step-byte-max': 3,
'terminal-cache': 0, 'terminal-cache': 0,
'filemanager-cache': 1, 'filemanager-cache': 1,
'upload-fragment': '500', 'upload-fragment': '500',
...@@ -334,7 +337,60 @@ class Form { ...@@ -334,7 +337,60 @@ class Form {
}, { }, {
type: "checkbox", name: 'use-multipart', label: LANG['list']['otherConf']['usemultipart'], type: "checkbox", name: 'use-multipart', label: LANG['list']['otherConf']['usemultipart'],
checked: opt['use-multipart'] === 1 checked: opt['use-multipart'] === 1
}, { }, { type: 'fieldset', offsetLeft: 0, label: LANG['list']['otherConf']['chunk']['title'], list: [
{ type: 'block', offsetLeft: 0, list: [
{
type: "checkbox", name: 'use-chunk', label: LANG['list']['otherConf']['chunk']['usechunk'], checked: opt['use-chunk'] === 1
},
]},
{ type: 'block', offsetLeft: 0, list: [
{ type:'label', label: LANG['list']['otherConf']['chunk']['min']},
{ type:'newcolumn' },
{
type: 'combo', label: '/byte', validate: 'ValidNumeric', inputWidth: 50, name: "chunk-step-byte-min",
options: ((items) => {
let ret = [];
// 如果自定义的路径不在items里，则++
if (items.indexOf(opt['chunk-step-byte-min']) === -1) {
items.unshift(opt['chunk-step-byte-min']);
}
items.map((_) => {
ret.push({
text: _,
value: _,
selected: opt['chunk-step-byte-min'] === _
})
});
return ret;
})([
'2', '4', '10', '50', '100', '500'
])
},
{ type:'newcolumn',},
{ type:'label', label: LANG['list']['otherConf']['chunk']['max'], offsetLeft: 30,},
{ type:'newcolumn' },
{
type: 'combo', label: '/byte', validate: 'ValidNumeric', inputWidth: 50, name: "chunk-step-byte-max",
options: ((items) => {
let ret = [];
// 如果自定义的路径不在items里，则++
if (items.indexOf(opt['chunk-step-byte-max']) === -1) {
items.unshift(opt['chunk-step-byte-max']);
}
items.map((_) => {
ret.push({
text: _,
value: _,
selected: opt['chunk-step-byte-max'] === _
})
});
return ret;
})([
'2', '4', '10', '50', '100', '500'
])
},
]},
]}, {
type: "checkbox", name: 'terminal-cache', label: LANG['list']['otherConf']['terminalCache'], type: "checkbox", name: 'terminal-cache', label: LANG['list']['otherConf']['terminalCache'],
checked: opt['terminal-cache'] === 1 checked: opt['terminal-cache'] === 1
}, { }, {
...@@ -405,6 +461,28 @@ class Form { ...@@ -405,6 +461,28 @@ class Form {
]) ])
} }
]}], true); ]}], true);
form.attachEvent('onChange', (name, value, state)=>{
switch(name){
case 'use-multipart':
if(state == true && form.isItemChecked('use-chunk')) {
form.uncheckItem('use-chunk');
}
break;
case 'use-chunk':
if(state == true && form.isItemChecked('use-multipart')) {
form.uncheckItem('use-multipart');
}
if(state == true) {
layer.open({
title: LANG_T['info']
,content: LANG['list']['otherConf']['chunk']['exphint']
});
}
break;
default:
break;
}
});
return form; return form;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment