(Enhancement:Modules:Request)优化返回包解码流程,增加自动猜解编码功能

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 ## `v(2.0.4-dev)`
 
+### 后端模块
+
+ * 优化返回包解码流程,增加自动猜解编码功能,该功能在中文较少时容易出错,出错了使用用户设置的字符编码
+
 ### Other
 
  * 新增 Python2 Custom CGI shell 示例

README.md

-# AntSword [![release](https://img.shields.io/badge/release-v2.0.3-blue.svg?style=flat-square)][url-release]
+# AntSword [![release](https://img.shields.io/badge/release-v2.0.3.1-blue.svg?style=flat-square)][url-release]
 
 > AntSword in your hands, no worries in your mind！
 

README_CN.md

-# 中国蚁剑 [![release](https://img.shields.io/badge/release-v2.0.3-blue.svg?style=flat-square)][url-release]
+# 中国蚁剑 [![release](https://img.shields.io/badge/release-v2.0.3.1-blue.svg?style=flat-square)][url-release]
 
 > 一剑在手，纵横无忧！
 

modules/request.js

@@ -7,6 +7,7 @@
 
 const fs = require('fs'),
   iconv = require('iconv-lite'),
+  jschardet = require('jschardet'),
   through = require('through'),
   CONF = require('./config'),
   superagent = require('superagent'),
@@ -135,14 +136,20 @@ class Request {
         }
         let buff = ret.hasOwnProperty('body') ? ret.body : new Buffer();
         // 解码
-        let text = iconv.decode(buff, opts['encode']);
+        let text = "";
+        // 自动猜测编码
+        let encoding = detectEncoding(buff, {defaultEncoding:"unknown"});
+        logger.debug("detect encoding:", encoding);
+        encoding = encoding != "unknown" ? encoding : opts['encode'];
+        text = iconv.decode(buff, encoding);
         if (err && text == "") {
           return event.sender.send('request-error-' + opts['hash'], err);
         };
         // 回调数据
         event.sender.send('request-' + opts['hash'], {
           text: text,
-          buff: buff
+          buff: buff,
+          encoding: encoding
         });
       });
   }
@@ -280,4 +287,43 @@ class Request {
 
 }
 
+/**
+ * 判断指定buffer对象的字符编码
+ * ref: https://github.com/LeoYuan/leoyuan.github.io/issues/25
+ * @param buffer
+ * @param options
+ *  - defaultEncoding 指定默认编码集
+ *  - minConfidence   指定可接受的最小confidence，如果判断结果小于此值，则用defaultEncoding
+ *  - verbose         返回更加详细的字符编码数据
+ * @returns {*}
+ */
+function detectEncoding(buffer, options) {
+
+  options = options || {};
+  buffer = buffer || Buffer('');
+
+  var DEFAULT_ENCODING = 'GBK', MIN_CONFIDENCE = 0.96;
+  var verbose = options.verbose;
+  var defaultEncoding = options.defaultEncoding || DEFAULT_ENCODING;
+  var minConfidence = options.minConfidence || MIN_CONFIDENCE;
+  var ret = jschardet.detect(buffer), encoding = ret.encoding === 'ascii' ? 'utf-8' : ret.encoding,
+      confidence = ret.confidence;
+  // var VALID_ENCODINGS = ['gb2312', 'gbk', 'utf-8', 'big5', 'euc-kr','euc-jp'];
+
+  if (encoding === null || !iconv.encodingExists(encoding) || confidence < minConfidence) {
+      return verbose ? {
+          encoding: defaultEncoding,
+          oriEncoding: encoding,
+          confidence: confidence
+      } : defaultEncoding;
+  } else {
+      encoding = encoding.toUpperCase();
+      return verbose ? {
+          encoding: encoding,
+          oriEncoding: encoding,
+          confidence: confidence
+      } : encoding;
+  }
+};
+
 module.exports = Request;

package-lock.json

 {
   "name": "antsword",
-  "version": "2.0.3",
+  "version": "2.0.3.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -446,6 +446,11 @@
       "resolved": "http://registry.npm.taobao.org/isarray/download/isarray-1.0.0.tgz",
       "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
     },
+    "jschardet": {
+      "version": "1.6.0",
+      "resolved": "http://registry.npm.taobao.org/jschardet/download/jschardet-1.6.0.tgz",
+      "integrity": "sha1-x9GnHtz/KDnbL57DD8XV69PBpng="
+    },
     "jstransform": {
       "version": "3.0.0",
       "resolved": "https://registry.npm.taobao.org/jstransform/download/jstransform-3.0.0.tgz",

package.json

 {
   "name": "antsword",
-  "version": "2.0.3",
+  "version": "2.0.3.1",
   "description": "中国蚁剑是一款跨平台的开源网站管理工具",
   "main": "app.js",
   "dependencies": {
     "extract-zip": "^1.6.7",
     "geoips": "0.0.1",
     "iconv-lite": "^0.4.23",
+    "jschardet": "^1.6.0",
     "nedb": "^1.5.1",
     "superagent": "^3.8.3",
     "superagent-proxy": "^1.0.3",

source/core/base.js

@@ -223,6 +223,7 @@ class Base {
         // 请求完毕返回数据{text,buff}
         .once(`request-${hash}`, (event, ret) => {
           return res({
+            'encoding': ret['encoding']||"",
             'text': ret['text'],
             'buff': ret['buff']
           });

source/modules/filemanager/index.js

@@ -895,6 +895,12 @@ class FileManager {
     ).then((res) => {
       let ret = res['text'];
       codes = ret;
+      let encoding = res['encoding'] || this.opts['encode'];
+      if(encoding.toUpperCase() == "UTF-8") {
+        encoding = "UTF8";
+      }
+      toolbar.setListOptionSelected('encode', `encode_${encoding}`);
+
       win.progressOff();
 
       // 初始化编辑器