From 960dbc5b35bc9a14b9552053a90a958829da13b5 Mon Sep 17 00:00:00 2001 From: Hsiao-nan Cheung Date: Fri, 9 Aug 2024 18:47:49 +0800 Subject: [PATCH] refactor: Rearrange functions --- lib/download.ps1 | 677 ++++++++++++++++++----------------- libexec/scoop-info.ps1 | 3 +- libexec/scoop-virustotal.ps1 | 5 - 3 files changed, 351 insertions(+), 334 deletions(-) diff --git a/lib/download.ps1 b/lib/download.ps1 index 7855b0b7e2..73ec12a48c 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -1,5 +1,55 @@ # Description: Functions for downloading files +## Meta downloader + +function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { + # we only want to show this warning once + if (!$use_cache) { warn 'Cache is being ignored.' } + + # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work + $urls = @(script:url $manifest $architecture) + + # can be multiple cookies: they will be used for all HTTP requests. + $cookies = $manifest.cookie + + # download first + if (Test-Aria2Enabled) { + Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash + } else { + foreach ($url in $urls) { + $fname = url_filename $url + + try { + Invoke-CachedDownload $app $version $url "$dir\$fname" $cookies $use_cache + } catch { + Write-Host -ForegroundColor DarkRed $_ + abort "URL $url is not valid" + } + + if ($check_hash) { + $manifest_hash = hash_for_url $manifest $url $architecture + $ok, $err = check_hash "$dir\$fname" $manifest_hash $(show_app $app $bucket) + if (!$ok) { + error $err + $cached = cache_path $app $version $url + if (Test-Path $cached) { + # rm cached file + Remove-Item -Force $cached + } + if ($url.Contains('sourceforge.net')) { + Write-Host -ForegroundColor Yellow 'SourceForge.net is known for causing hash validation fails. Please try again before opening a ticket.' + } + abort $(new_issue_msg $app $bucket 'hash check failed') + } + } + } + } + + return $urls.ForEach({ url_filename $_ }) +} + +## [System.Net] downloader + function Invoke-CachedDownload ($app, $version, $url, $to, $cookies = $null, $use_cache = $true) { $cached = cache_path $app $version $url @@ -35,6 +85,183 @@ function Start-Download ($url, $to, $cookies) { } } +function Invoke-Download ($url, $to, $cookies, $progress) { + # download with filesize and progress indicator + $reqUrl = ($url -split '#')[0] + $wreq = [Net.WebRequest]::Create($reqUrl) + if ($wreq -is [Net.HttpWebRequest]) { + $wreq.UserAgent = Get-UserAgent + if (-not ($url -match 'sourceforge\.net' -or $url -match 'portableapps\.com')) { + $wreq.Referer = strip_filename $url + } + if ($url -match 'api\.github\.com/repos') { + $wreq.Accept = 'application/octet-stream' + $wreq.Headers['Authorization'] = "Bearer $(Get-GitHubToken)" + $wreq.Headers['X-GitHub-Api-Version'] = '2022-11-28' + } + if ($cookies) { + $wreq.Headers.Add('Cookie', (cookie_header $cookies)) + } + + get_config PRIVATE_HOSTS | Where-Object { $_ -ne $null -and $url -match $_.match } | ForEach-Object { + (ConvertFrom-StringData -StringData $_.Headers).GetEnumerator() | ForEach-Object { + $wreq.Headers[$_.Key] = $_.Value + } + } + } + + try { + $wres = $wreq.GetResponse() + } catch [System.Net.WebException] { + $exc = $_.Exception + $handledCodes = @( + [System.Net.HttpStatusCode]::MovedPermanently, # HTTP 301 + [System.Net.HttpStatusCode]::Found, # HTTP 302 + [System.Net.HttpStatusCode]::SeeOther, # HTTP 303 + [System.Net.HttpStatusCode]::TemporaryRedirect # HTTP 307 + ) + + # Only handle redirection codes + $redirectRes = $exc.Response + if ($handledCodes -notcontains $redirectRes.StatusCode) { + throw $exc + } + + # Get the new location of the file + if ((-not $redirectRes.Headers) -or ($redirectRes.Headers -notcontains 'Location')) { + throw $exc + } + + $newUrl = $redirectRes.Headers['Location'] + info "Following redirect to $newUrl..." + + # Handle manual file rename + if ($url -like '*#/*') { + $null, $postfix = $url -split '#/' + $newUrl = "$newUrl#/$postfix" + } + + Invoke-Download $newUrl $to $cookies $progress + return + } + + $total = $wres.ContentLength + if ($total -eq -1 -and $wreq -is [net.ftpwebrequest]) { + $total = ftp_file_size($url) + } + + if ($progress -and ($total -gt 0)) { + [console]::CursorVisible = $false + function Trace-DownloadProgress ($read) { + Write-DownloadProgress $read $total $url + } + } else { + Write-Host "Downloading $url ($(filesize $total))..." + function Trace-DownloadProgress { + #no op + } + } + + try { + $s = $wres.getresponsestream() + $fs = [io.file]::openwrite($to) + $buffer = New-Object byte[] 2048 + $totalRead = 0 + $sw = [diagnostics.stopwatch]::StartNew() + + Trace-DownloadProgress $totalRead + while (($read = $s.read($buffer, 0, $buffer.length)) -gt 0) { + $fs.write($buffer, 0, $read) + $totalRead += $read + if ($sw.elapsedmilliseconds -gt 100) { + $sw.restart() + Trace-DownloadProgress $totalRead + } + } + $sw.stop() + Trace-DownloadProgress $totalRead + } finally { + if ($progress) { + [console]::CursorVisible = $true + Write-Host + } + if ($fs) { + $fs.close() + } + if ($s) { + $s.close() + } + $wres.close() + } +} + +function Format-DownloadProgress ($url, $read, $total, $console) { + $filename = url_remote_filename $url + + # calculate current percentage done + $p = [math]::Round($read / $total * 100, 0) + + # pre-generate LHS and RHS of progress string + # so we know how much space we have + $left = "$filename ($(filesize $total))" + $right = [string]::Format('{0,3}%', $p) + + # calculate remaining width for progress bar + $midwidth = $console.BufferSize.Width - ($left.Length + $right.Length + 8) + + # calculate how many characters are completed + $completed = [math]::Abs([math]::Round(($p / 100) * $midwidth, 0) - 1) + + # generate dashes to symbolise completed + if ($completed -gt 1) { + $dashes = [string]::Join('', ((1..$completed) | ForEach-Object { '=' })) + } + + # this is why we calculate $completed - 1 above + $dashes += switch ($p) { + 100 { '=' } + default { '>' } + } + + # the remaining characters are filled with spaces + $spaces = switch ($dashes.Length) { + $midwidth { [string]::Empty } + default { + [string]::Join('', ((1..($midwidth - $dashes.Length)) | ForEach-Object { ' ' })) + } + } + + "$left [$dashes$spaces] $right" +} + +function Write-DownloadProgress ($read, $total, $url) { + $console = $Host.UI.RawUI + $left = $console.CursorPosition.X + $top = $console.CursorPosition.Y + $width = $console.BufferSize.Width + + if ($read -eq 0) { + $maxOutputLength = $(Format-DownloadProgress $url 100 $total $console).Length + if (($left + $maxOutputLength) -gt $width) { + # not enough room to print progress on this line + # print on new line + Write-Host + $left = 0 + $top = $top + 1 + if ($top -gt $console.CursorPosition.Y) { $top = $console.CursorPosition.Y } + } + } + + Write-Host $(Format-DownloadProgress $url $read $total $console) -NoNewline + [console]::SetCursorPosition($left, $top) +} + +## Aria2 downloader + +function Test-Aria2Enabled { + return (Test-HelperInstalled -Helper Aria2) -and (get_config 'aria2-enabled' $true) +} + function aria_exit_code($exitcode) { $codes = @{ 0 = 'All downloads were successful' @@ -164,7 +391,7 @@ function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $ if ((Test-Path $data.$url.source) -and -not((Test-Path "$($data.$url.source).aria2") -or (Test-Path $urlstxt)) -and $use_cache) { Write-Host 'Loading ' -NoNewline - Write-Host $(url_remote_filename $url) -f Cyan -NoNewline + Write-Host $(url_remote_filename $url) -ForegroundColor Cyan -NoNewline Write-Host ' from cache.' } else { $download_finished = $false @@ -284,225 +511,12 @@ function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $ } } -# download with filesize and progress indicator -function Invoke-Download ($url, $to, $cookies, $progress) { - $reqUrl = ($url -split '#')[0] - $wreq = [Net.WebRequest]::Create($reqUrl) - if ($wreq -is [Net.HttpWebRequest]) { - $wreq.UserAgent = Get-UserAgent - if (-not ($url -match 'sourceforge\.net' -or $url -match 'portableapps\.com')) { - $wreq.Referer = strip_filename $url - } - if ($url -match 'api\.github\.com/repos') { - $wreq.Accept = 'application/octet-stream' - $wreq.Headers['Authorization'] = "Bearer $(Get-GitHubToken)" - $wreq.Headers['X-GitHub-Api-Version'] = '2022-11-28' - } - if ($cookies) { - $wreq.Headers.Add('Cookie', (cookie_header $cookies)) - } +## Helper functions - get_config PRIVATE_HOSTS | Where-Object { $_ -ne $null -and $url -match $_.match } | ForEach-Object { - (ConvertFrom-StringData -StringData $_.Headers).GetEnumerator() | ForEach-Object { - $wreq.Headers[$_.Key] = $_.Value - } - } - } +### Downloader parameters - try { - $wres = $wreq.GetResponse() - } catch [System.Net.WebException] { - $exc = $_.Exception - $handledCodes = @( - [System.Net.HttpStatusCode]::MovedPermanently, # HTTP 301 - [System.Net.HttpStatusCode]::Found, # HTTP 302 - [System.Net.HttpStatusCode]::SeeOther, # HTTP 303 - [System.Net.HttpStatusCode]::TemporaryRedirect # HTTP 307 - ) - - # Only handle redirection codes - $redirectRes = $exc.Response - if ($handledCodes -notcontains $redirectRes.StatusCode) { - throw $exc - } - - # Get the new location of the file - if ((-not $redirectRes.Headers) -or ($redirectRes.Headers -notcontains 'Location')) { - throw $exc - } - - $newUrl = $redirectRes.Headers['Location'] - info "Following redirect to $newUrl..." - - # Handle manual file rename - if ($url -like '*#/*') { - $null, $postfix = $url -split '#/' - $newUrl = "$newUrl#/$postfix" - } - - Invoke-Download $newUrl $to $cookies $progress - return - } - - $total = $wres.ContentLength - if ($total -eq -1 -and $wreq -is [net.ftpwebrequest]) { - $total = ftp_file_size($url) - } - - if ($progress -and ($total -gt 0)) { - [console]::CursorVisible = $false - function Trace-DownloadProgress ($read) { - Write-DownloadProgress $read $total $url - } - } else { - Write-Host "Downloading $url ($(filesize $total))..." - function Trace-DownloadProgress { - #no op - } - } - - try { - $s = $wres.getresponsestream() - $fs = [io.file]::openwrite($to) - $buffer = New-Object byte[] 2048 - $totalRead = 0 - $sw = [diagnostics.stopwatch]::StartNew() - - Trace-DownloadProgress $totalRead - while (($read = $s.read($buffer, 0, $buffer.length)) -gt 0) { - $fs.write($buffer, 0, $read) - $totalRead += $read - if ($sw.elapsedmilliseconds -gt 100) { - $sw.restart() - Trace-DownloadProgress $totalRead - } - } - $sw.stop() - Trace-DownloadProgress $totalRead - } finally { - if ($progress) { - [console]::CursorVisible = $true - Write-Host - } - if ($fs) { - $fs.close() - } - if ($s) { - $s.close() - } - $wres.close() - } -} - -function Format-DownloadProgress ($url, $read, $total, $console) { - $filename = url_remote_filename $url - - # calculate current percentage done - $p = [math]::Round($read / $total * 100, 0) - - # pre-generate LHS and RHS of progress string - # so we know how much space we have - $left = "$filename ($(filesize $total))" - $right = [string]::Format('{0,3}%', $p) - - # calculate remaining width for progress bar - $midwidth = $console.BufferSize.Width - ($left.Length + $right.Length + 8) - - # calculate how many characters are completed - $completed = [math]::Abs([math]::Round(($p / 100) * $midwidth, 0) - 1) - - # generate dashes to symbolise completed - if ($completed -gt 1) { - $dashes = [string]::Join('', ((1..$completed) | ForEach-Object { '=' })) - } - - # this is why we calculate $completed - 1 above - $dashes += switch ($p) { - 100 { '=' } - default { '>' } - } - - # the remaining characters are filled with spaces - $spaces = switch ($dashes.Length) { - $midwidth { [string]::Empty } - default { - [string]::Join('', ((1..($midwidth - $dashes.Length)) | ForEach-Object { ' ' })) - } - } - - "$left [$dashes$spaces] $right" -} - -function Write-DownloadProgress ($read, $total, $url) { - $console = $host.UI.RawUI - $left = $console.CursorPosition.X - $top = $console.CursorPosition.Y - $width = $console.BufferSize.Width - - if ($read -eq 0) { - $maxOutputLength = $(Format-DownloadProgress $url 100 $total $console).length - if (($left + $maxOutputLength) -gt $width) { - # not enough room to print progress on this line - # print on new line - Write-Host - $left = 0 - $top = $top + 1 - if ($top -gt $console.CursorPosition.Y) { $top = $console.CursorPosition.Y } - } - } - - Write-Host $(Format-DownloadProgress $url $read $total $console) -NoNewline - [console]::SetCursorPosition($left, $top) -} - -function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { - # we only want to show this warning once - if (!$use_cache) { warn 'Cache is being ignored.' } - - # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work - $urls = @(script:url $manifest $architecture) - - # can be multiple cookies: they will be used for all HTTP requests. - $cookies = $manifest.cookie - - # download first - if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash - } else { - foreach ($url in $urls) { - $fname = url_filename $url - - try { - Invoke-CachedDownload $app $version $url "$dir\$fname" $cookies $use_cache - } catch { - Write-Host -f darkred $_ - abort "URL $url is not valid" - } - - if ($check_hash) { - $manifest_hash = hash_for_url $manifest $url $architecture - $ok, $err = check_hash "$dir\$fname" $manifest_hash $(show_app $app $bucket) - if (!$ok) { - error $err - $cached = cache_path $app $version $url - if (Test-Path $cached) { - # rm cached file - Remove-Item -Force $cached - } - if ($url.Contains('sourceforge.net')) { - Write-Host -f yellow 'SourceForge.net is known for causing hash validation fails. Please try again before opening a ticket.' - } - abort $(new_issue_msg $app $bucket 'hash check failed') - } - } - } - } - - return $urls.ForEach({ url_filename $_ }) -} - -function cookie_header($cookies) { - if (!$cookies) { return } +function cookie_header($cookies) { + if (!$cookies) { return } $vals = $cookies.psobject.properties | ForEach-Object { "$($_.name)=$($_.value)" @@ -511,91 +525,51 @@ function cookie_header($cookies) { [string]::join(';', $vals) } -function ftp_file_size($url) { - $request = [net.ftpwebrequest]::create($url) - $request.method = [net.webrequestmethods+ftp]::getfilesize - $request.getresponse().contentlength +function Get-Encoding($wc) { + if ($null -ne $wc.ResponseHeaders -and $wc.ResponseHeaders['Content-Type'] -match 'charset=([^;]*)') { + return [System.Text.Encoding]::GetEncoding($Matches[1]) + } else { + return [System.Text.Encoding]::GetEncoding('utf-8') + } } -# hashes -function hash_for_url($manifest, $url, $arch) { - $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } - - if ($hashes.length -eq 0) { return $null } - - $urls = @(script:url $manifest $arch) - - $index = [array]::indexof($urls, $url) - if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } - - @($hashes)[$index] +function Get-UserAgent() { + return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if(${env:ProgramFiles(Arm)}){'ARM64; '}elseif($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -in 'AMD64','ARM64'){'WOW64; '})$PSEdition)" } -# returns (ok, err) -function check_hash($file, $hash, $app_name) { - if (!$hash) { - warn "Warning: No hash in manifest. SHA256 for '$(fname $file)' is:`n $((Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower())" - return $true, $null - } - - Write-Host 'Checking hash of ' -NoNewline - Write-Host $(url_remote_filename $url) -f Cyan -NoNewline - Write-Host ' ... ' -NoNewline - $algorithm, $expected = get_hash $hash - if ($null -eq $algorithm) { - return $false, "Hash type '$algorithm' isn't supported." +function setup_proxy() { + # note: '@' and ':' in password must be escaped, e.g. 'p@ssword' -> p\@ssword' + $proxy = get_config PROXY + if (!$proxy) { + return } - - $actual = (Get-FileHash -Path $file -Algorithm $algorithm).Hash.ToLower() - $expected = $expected.ToLower() - - if ($actual -ne $expected) { - $msg = "Hash check failed!`n" - $msg += "App: $app_name`n" - $msg += "URL: $url`n" - if (Test-Path $file) { - $msg += "First bytes: $((get_magic_bytes_pretty $file ' ').ToUpper())`n" - } - if ($expected -or $actual) { - $msg += "Expected: $expected`n" - $msg += "Actual: $actual" + try { + $credentials, $address = $proxy -split '(?.*)(?:\/|\?dwl=)(?.*)$') { @@ -644,6 +620,20 @@ function handle_special_urls($url) { return $url } +### Remote file infomation + +function download_json($url) { + $githubtoken = Get-GitHubToken + $authheader = @{} + if ($githubtoken) { + $authheader = @{'Authorization' = "token $githubtoken" } + } + $ProgressPreference = 'SilentlyContinue' + $result = Invoke-WebRequest $url -UseBasicParsing -Headers $authheader | Select-Object -ExpandProperty content | ConvertFrom-Json + $ProgressPreference = 'Continue' + $result +} + function get_magic_bytes($file) { if (!(Test-Path $file)) { return '' @@ -665,49 +655,17 @@ function get_magic_bytes_pretty($file, $glue = ' ') { return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue } -function Get-Encoding($wc) { - if ($null -ne $wc.ResponseHeaders -and $wc.ResponseHeaders['Content-Type'] -match 'charset=([^;]*)') { - return [System.Text.Encoding]::GetEncoding($Matches[1]) - } else { - return [System.Text.Encoding]::GetEncoding('utf-8') - } -} - -function Get-UserAgent() { - return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if(${env:ProgramFiles(Arm)}){'ARM64; '}elseif($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -in 'AMD64','ARM64'){'WOW64; '})$PSEdition)" -} - -function setup_proxy() { - # note: '@' and ':' in password must be escaped, e.g. 'p@ssword' -> p\@ssword' - $proxy = get_config PROXY - if (!$proxy) { - return - } - try { - $credentials, $address = $proxy -split '(?