fix(core): add Get-Encoding function to fix missing webClient encod…

…ing (ScoopInstaller#4956) Add `Get-Encoding` function in core.ps1 closes: ScoopInstaller#4911 ScoopInstaller#4324 Co-authored-by: Hsiao-nan Cheung <[email protected]>
yi-Xu-0100 · Jun 9, 2022 · bf77676 · bf77676
1 parent 3a1186e
commit bf77676
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,7 +10,8 @@
 - **chore:** Update Nonportable bucket URL ([#4955](https://github.com/ScoopInstaller/Scoop/issues/4955))
 - **core:** Using `Invoke-Command` instead of `Invoke-Expression` ([#4941](https://github.com/ScoopInstaller/Scoop/issues/4941))
 - **core:** Load config file before initialization ([#4932](https://github.com/ScoopInstaller/Scoop/issues/4932))
-- **core:** Allow to use '_' and '.' in bucket name ([#4952](https://github.com/ScoopInstaller/Scoop/issues/4952))
+- **core:** Allow to use '_' and '.' in bucket name ([#4952](https://github.com/ScoopInstaller/Scoop/pull/4952))
+- **core:** Add `Get-Encoding` function to fix missing webclient encoding ([#4956](https://github.com/ScoopInstaller/Scoop/pull/4956))
 - **depends:** Avoid digits in archive file extension (except for .7z and .001) ([#4915](https://github.com/ScoopInstaller/Scoop/issues/4915))
 - **bucket:** Don't check remote URL of non-git buckets ([#4923](https://github.com/ScoopInstaller/Scoop/issues/4923))
 - **bucket:** Don't write message OK before bucket is cloned ([#4925](https://github.com/ScoopInstaller/Scoop/issues/4925))

diff --git a/bin/checkver.ps1 b/bin/checkver.ps1
@@ -113,7 +113,7 @@ $Queue | ForEach-Object {
     } else {
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
     }
-    Register-ObjectEvent $wc downloadstringcompleted -ErrorAction Stop | Out-Null
+    Register-ObjectEvent $wc downloadDataCompleted -ErrorAction Stop | Out-Null
 
     $githubRegex = '\/releases\/tag\/(?:v|V)?([\d.]+)'
 
@@ -190,7 +190,7 @@ $Queue | ForEach-Object {
     }
 
     $wc.Headers.Add('Referer', (strip_filename $url))
-    $wc.DownloadStringAsync($url, $state)
+    $data = $wc.DownloadDataAsync($url, $state)
 }
 
 function next($er) {
@@ -218,7 +218,7 @@ while ($in_progress -gt 0) {
     $ver = $Version
 
     if (!$ver) {
-        $page = $ev.SourceEventArgs.Result
+        $page = (Get-Encoding($wc)).GetString($ev.SourceEventArgs.Result)
         $err = $ev.SourceEventArgs.Error
         if ($json.checkver.script) {
             $page = Invoke-Command ([scriptblock]::Create($json.checkver.script -join "`r`n"))

diff --git a/bin/describe.ps1 b/bin/describe.ps1
@@ -44,7 +44,8 @@ $Queue | ForEach-Object {
     try {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $home_html = $wc.DownloadString($manifest.homepage)
+        $data = $wc.DownloadData($manifest.homepage)
+        $home_html = (Get-Encoding($wc)).GetString($data)
     } catch {
         Write-Host "`n$($_.Exception.Message)" -ForegroundColor Red
         return

diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1
@@ -1,20 +1,21 @@
 # Must included with 'json.ps1'
 function find_hash_in_rdf([String] $url, [String] $basename) {
-    $data = $null
+    $xml = $null
     try {
         # Download and parse RDF XML file
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        [xml]$data = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        [xml]$xml = (Get-Encoding($wc)).GetString($data)
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"
         return $null
     }
 
     # Find file content
-    $digest = $data.RDF.Content | Where-Object { [String]$_.about -eq $basename }
+    $digest = $xml.RDF.Content | Where-Object { [String]$_.about -eq $basename }
 
     return format_hash $digest.sha256
 }
@@ -35,7 +36,8 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $hashfile = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $hashfile = (Get-Encoding($wc)).GetString($data)
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"
@@ -88,7 +90,8 @@ function find_hash_in_json([String] $url, [Hashtable] $substitutions, [String] $
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $json = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $json = (Get-Encoding($wc)).GetString($data)
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"
@@ -108,7 +111,8 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $xml = [xml]$wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $xml = [xml]((Get-Encoding($wc)).GetString($data))
     } catch [system.net.webexception] {
         write-host -f darkred $_
         write-host -f darkred "URL $url is not valid"

diff --git a/lib/core.ps1 b/lib/core.ps1
@@ -15,6 +15,14 @@ function Optimize-SecurityProtocol {
     }
 }
 
+function Get-Encoding($wc) {
+    if($wc.ResponseHeaders["Content-Type"] -match 'charset=([^;]*)') {
+        return [System.Text.Encoding]::GetEncoding($Matches[1])
+    } else {
+        return [System.Text.Encoding]::GetEncoding('utf-8')
+    }
+}
+
 function Get-UserAgent() {
     return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -eq 'AMD64'){'WOW64; '})$PSEdition)"
 }

diff --git a/lib/description.ps1 b/lib/description.ps1
@@ -18,7 +18,8 @@ function find_description($url, $html, $redir = $false) {
     if($refresh -and !$redir) {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $html = $wc.downloadstring($refresh)
+        $data = $wc.DownloadData($refresh)
+        $html = (Get-Encoding($wc)).GetString($data)
         return find_description $refresh $html $true
     }
 

diff --git a/lib/manifest.ps1 b/lib/manifest.ps1
@@ -12,7 +12,8 @@ function url_manifest($url) {
     try {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $str = $wc.downloadstring($url)
+        $data = $wc.DownloadData($url)
+        $str = (Get-Encoding($wc)).GetString($data)
     } catch [system.management.automation.methodinvocationexception] {
         warn "error: $($_.exception.innerexception.message)"
     } catch {
@@ -65,7 +66,8 @@ function save_installed_manifest($app, $bucket, $dir, $url) {
     if ($url) {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
-        $wc.downloadstring($url) > "$dir\manifest.json"
+        $data = $wc.DownloadData($url)
+        (Get-Encoding($wc)).GetString($data) > "$dir\manifest.json"
     } else {
         Copy-Item (manifest_path $app $bucket) "$dir\manifest.json"
     }

diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1
@@ -85,7 +85,8 @@ Function Get-VirusTotalResult($hash, $app) {
     $url = "https://www.virustotal.com/ui/files/$hash"
     $wc = New-Object Net.Webclient
     $wc.Headers.Add('User-Agent', (Get-UserAgent))
-    $result = $wc.downloadstring($url)
+    $data = $wc.DownloadData($url)
+    $result = (Get-Encoding($wc)).GetString($data)
     $stats = json_path $result '$.data.attributes.last_analysis_stats'
     $malicious = json_path $stats '$.malicious'
     $suspicious = json_path $stats '$.suspicious'