feat: added new experimental encoding detection (#831)

Author: JohnstonCode

package.json

@@ -49,6 +49,7 @@
         "watch:css": "yarn run build:css -w"
     },
     "dependencies": {
+        "chardet": "^0.8.0",
         "dayjs": "^1.8.20",
         "minimatch": "^3.0.4",
         "original-fs": "^1.0.0",
@@ -59,6 +60,7 @@
     "devDependencies": {
         "@semantic-release/changelog": "^5.0.0",
         "@semantic-release/git": "^9.0.0",
+        "@types/chardet": "^0.8.0",
         "@types/glob": "^7.1.1",
         "@types/mocha": "^7.0.0",
         "@types/node": "^12.11.7",
@@ -1246,6 +1248,11 @@
                     "type": "boolean",
                     "description": "Set to ignore externals definitions on update (add --ignore-externals)",
                     "default": true
+                },
+                "svn.experimental.detect_encoding": {
+                    "type": "boolean",
+                    "description": "Try the experimental encoding detection",
+                    "default": false
                 }
             }
         }

src/encoding.ts

@@ -1,4 +1,6 @@
 import { jschardet } from "./vscodeModules";
+import * as chardet from "chardet";
+import { configuration } from "./helpers/configuration";
 
 if (jschardet.Constants) {
   jschardet.Constants.MINIMUM_THRESHOLD = 0.2;
@@ -51,7 +53,20 @@ export function detectEncoding(buffer: Buffer): string | null {
     return result;
   }
 
-  const detected = jschardet.detect(buffer);
+  const experimental = configuration.get<boolean>(
+    "experimental.detect_encoding",
+    false
+  );
+  if (experimental) {
+    const detected = chardet.detect(buffer);
+    if (detected) {
+      return detected.replace(/[^a-zA-Z0-9]/g, "").toLocaleLowerCase();
+    }
+
+    return null;
+  }
+
+  const detected = jschardet.detect(buffer.slice(0, 512 * 128)); // ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
 
   if (!detected || !detected.encoding || detected.confidence < 0.8) {
     return null;

yarn.lock

@@ -247,6 +247,13 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.0.0.tgz#9c13c2574c92d4503b005feca8f2e16cc1611506"
   integrity sha512-KYyTT/T6ALPkIRd2Ge080X/BsXvy9O0hcWTtMWkPvwAwF99+vn6Dv4GzrFT/Nn1LePr+FFDbRXXlqmsy9lw2zA==
 
+"@types/chardet@^0.8.0":
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/@types/chardet/-/chardet-0.8.0.tgz#40932a9d751bb1ff22e7403312faaf81755ad432"
+  integrity sha512-0PFX0r+bt2W6np4tZzF2Gh28pPLTM7lgjLMs0DJnV/Y4rPI3kLJCwtwRTb4aGl1iHmuF8TEmizzfJOw/7XQRfw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/color-name@^1.1.1":
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
@@ -901,6 +908,11 @@ chardet@^0.7.0:
   resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
   integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==
 
+chardet@^0.8.0:
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.8.0.tgz#215e9e457296aa88fb0c38b010fd7a7e20482ed3"
+  integrity sha512-fRAe54sDSPvCz9I3puKUoUpLBEIUjlwBoNyNcD2eAiP5Ybw2iXnrT7w15hfkNywosXFNllWwvOKsxl7UUCKQaQ==
+
 cheerio@^1.0.0-rc.1:
   version "1.0.0-rc.3"
   resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.3.tgz#094636d425b2e9c0f4eb91a46c05630c9a1a8bf6"