From e21be4f24120fea4eb71589aff59106e5fb69a11 Mon Sep 17 00:00:00 2001 From: notPlancha Date: Mon, 5 Aug 2024 02:17:27 +0100 Subject: [PATCH 1/8] dont block script from homepage --- bin/checkver.ps1 | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index 57010c9992..5fec932345 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -279,25 +279,26 @@ while ($in_progress -gt 0) { next "'replace' requires 're' or 'regex'" continue } - $err = $ev.SourceEventArgs.Error - if ($err) { - next "$($err.message)`r`nURL $url is not valid" - continue - } - - if ($url) { - $ms = New-Object System.IO.MemoryStream - $ms.Write($result, 0, $result.Length) - $ms.Seek(0, 0) | Out-Null - if ($result[0] -eq 0x1F -and $result[1] -eq 0x8B) { - $ms = New-Object System.IO.Compression.GZipStream($ms, [System.IO.Compression.CompressionMode]::Decompress) - } - $page = (New-Object System.IO.StreamReader($ms, (Get-Encoding $wc))).ReadToEnd() - } - $source = $url if ($script) { $page = Invoke-Command ([scriptblock]::Create($script -join "`r`n")) $source = 'the output of script' + } else { + $err = $ev.SourceEventArgs.Error + if ($err) { + next "$($err.message)`r`nURL $url is not valid" + continue + } + + if ($url) { + $ms = New-Object System.IO.MemoryStream + $ms.Write($result, 0, $result.Length) + $ms.Seek(0, 0) | Out-Null + if ($result[0] -eq 0x1F -and $result[1] -eq 0x8B) { + $ms = New-Object System.IO.Compression.GZipStream($ms, [System.IO.Compression.CompressionMode]::Decompress) + } + $page = (New-Object System.IO.StreamReader($ms, (Get-Encoding $wc))).ReadToEnd() + } + $source = $url } if ($jsonpath) { From 1453b8474ed388ec0a365b9b7554d3e4370ba65b Mon Sep 17 00:00:00 2001 From: notPlancha Date: Mon, 5 Aug 2024 02:31:30 +0100 Subject: [PATCH 2/8] fix: Homepage no longer downloaded on script #5704 From 7f99c499d72d1717e477e9b90fb43693b4b7ca6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olav=20R=C3=B8nnestad=20Birkeland?= <6450056+o-l-a-v@users.noreply.github.com> Date: Fri, 9 Aug 2024 05:03:51 +0200 Subject: [PATCH 3/8] fix (decompress): `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty (#6092) Co-authored-by: Hsiao-nan Cheung --- CHANGELOG.md | 6 ++++++ lib/decompress.ps1 | 9 ++++++--- test/Scoop-Decompress.Tests.ps1 | 12 ++++++++++-- test/fixtures/decompress/TestCases.zip | Bin 532327 -> 532393 bytes 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f19e7662f4..b7f432537f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## [Unreleased](https://github.com/ScoopInstaller/Scoop/compare/master...develop) + +### Bug Fixes + +- **decompress**: `Expand-7zipArchive` Only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) + ## [v0.5.2](https://github.com/ScoopInstaller/Scoop/compare/v0.5.1...v0.5.2) - 2024-07-26 ### Bug Fixes diff --git a/lib/decompress.ps1 b/lib/decompress.ps1 index 9c3f5c6813..964c7ca5df 100644 --- a/lib/decompress.ps1 +++ b/lib/decompress.ps1 @@ -123,15 +123,18 @@ function Expand-7zipArchive { } if (!$IsTar -and $ExtractDir) { movedir "$DestinationPath\$ExtractDir" $DestinationPath | Out-Null - # Remove temporary directory - Remove-Item "$DestinationPath\$($ExtractDir -replace '[\\/].*')" -Recurse -Force -ErrorAction Ignore + # Remove temporary directory if it is empty + $ExtractDirTopPath = [string] "$DestinationPath\$($ExtractDir -replace '[\\/].*')" + if ((Get-ChildItem -Path $ExtractDirTopPath -Force -ErrorAction Ignore).Count -eq 0) { + Remove-Item -Path $ExtractDirTopPath -Recurse -Force -ErrorAction Ignore + } } if (Test-Path $LogPath) { Remove-Item $LogPath -Force } if ($Removal) { if (($Path -replace '.*\.([^\.]*)$', '$1') -eq '001') { - # Remove splited 7-zip archive parts + # Remove splitted 7-zip archive parts Get-ChildItem "$($Path -replace '\.[^\.]*$', '').???" | Remove-Item -Force } elseif (($Path -replace '.*\.part(\d+)\.rar$', '$1')[-1] -eq '1') { # Remove splitted RAR archive parts diff --git a/test/Scoop-Decompress.Tests.ps1 b/test/Scoop-Decompress.Tests.ps1 index 4ec31a4ece..e635492cec 100644 --- a/test/Scoop-Decompress.Tests.ps1 +++ b/test/Scoop-Decompress.Tests.ps1 @@ -25,7 +25,7 @@ Describe 'Decompression function' -Tag 'Scoop', 'Windows', 'Decompress' { } It 'Test cases should exist and hash should match' { $testcases | Should -Exist - (Get-FileHash -Path $testcases -Algorithm SHA256).Hash.ToLower() | Should -Be 'afb86b0552187b8d630ce25d02835fb809af81c584f07e54cb049fb74ca134b6' + (Get-FileHash -Path $testcases -Algorithm SHA256).Hash.ToLower() | Should -Be '591072faabd419b77932b7023e5899b4e05c0bf8e6859ad367398e6bfe1eb203' } It 'Test cases should be extracted correctly' { { Microsoft.PowerShell.Archive\Expand-Archive -Path $testcases -DestinationPath $working_dir } | Should -Not -Throw @@ -61,7 +61,7 @@ Describe 'Decompression function' -Tag 'Scoop', 'Windows', 'Decompress' { $to = test_extract 'Expand-7zipArchive' $test1 $to | Should -Exist "$to\empty" | Should -Exist - (Get-ChildItem $to).Count | Should -Be 3 + (Get-ChildItem $to).Count | Should -Be 4 } It 'extract "extract_dir" correctly' { @@ -78,6 +78,14 @@ Describe 'Decompression function' -Tag 'Scoop', 'Windows', 'Decompress' { (Get-ChildItem $to).Count | Should -Be 1 } + It 'extract "extract_dir" with nested folder with same name' { + $to = test_extract 'Expand-7zipArchive' $test1 $false 'keep\sub' + $to | Should -Exist + "$to\keep\empty" | Should -Exist + (Get-ChildItem $to).Count | Should -Be 1 + (Get-ChildItem "$to\keep").Count | Should -Be 1 + } + It 'extract nested compressed file' { # file ext: tgz $to = test_extract 'Expand-7zipArchive' $test2 diff --git a/test/fixtures/decompress/TestCases.zip b/test/fixtures/decompress/TestCases.zip index d2cd98c0f2340427f32cdbc031b6fcfd91f073d9..fb2092e56c1f231ed9b54c602953e744460e7093 100644 GIT binary patch delta 446 zcmaF9PhsVLh57(*W)=|!CI$|Ma}gYo+}Lvmhk3xWrJ;yvlg} zt<7<H9m^^pqQy z#4;|NcfIfKzfHj>E-lG;_4CK`;^b%kziRGZQ`nu_xNT=|eNv?Z%j2_4m-)8MH-rm$m!f(oz2T_jNv7p&6UoxwbmCvOG67x70K@w`&n>jBIT4 zj(O-Et1Nzh-N1l>A;7zFPwO7W);&zEdzf4Iu(a-BZQaAxx`%!19u5mhUX&1-&M3v< z&G=`!uM|fdW8?HSQXEN4d`!~?q&X58CrvMu=163`H~lh@V$x!sZYBd1&6_@5h9i~9 zmVNp^8IBYt9`5NWvK%Rlh0{;Ua>O#}uAZ(T#}Un>mN&gsjw7Cleb)46K)%|nS?mmc g!JfgOVAo5nNDc62Wdp_p13wUEvNAAeNpS#q0IXlLlmGw# delta 377 zcmZ3vU*Y*ah57(*W)=|!CI$|M-n}a#-sc49PhenRm<+@`Kx`hBSrC$1TwGJlUp}mhbaD>l)wwdYu zv+L#m%O{U^RzZqM9 zGqwI^ZvD;D`kS@&H(Tp(_N~7;tR;C-JTu){io=_6*YtiVjyOj9>F=aCl9eREOrLJV9#Jsi0P$Pqy~7ivH=5;fgcDn MSs55!NOAyq0H|AzO#lD@ From 79cf33d0b7d5d8ffc95e58cc248626d2f6096319 Mon Sep 17 00:00:00 2001 From: Hsiao-nan Cheung Date: Sun, 11 Aug 2024 17:23:59 +0800 Subject: [PATCH 4/8] refactor(download): Move download-related functions to 'download.ps1' (#6095) --- CHANGELOG.md | 6 +- bin/checkhashes.ps1 | 2 +- bin/checkurls.ps1 | 2 +- bin/checkver.ps1 | 2 +- bin/describe.ps1 | 1 + lib/autoupdate.ps1 | 13 + lib/core.ps1 | 176 -------- lib/download.ps1 | 758 ++++++++++++++++++++++++++++++++++ lib/install.ps1 | 566 ------------------------- libexec/scoop-download.ps1 | 2 +- libexec/scoop-info.ps1 | 3 +- libexec/scoop-install.ps1 | 1 + libexec/scoop-search.ps1 | 24 +- libexec/scoop-update.ps1 | 1 + libexec/scoop-virustotal.ps1 | 7 +- test/Scoop-Core.Tests.ps1 | 22 - test/Scoop-Download.Tests.ps1 | 49 +++ test/Scoop-Install.Tests.ps1 | 22 - 18 files changed, 836 insertions(+), 821 deletions(-) create mode 100644 lib/download.ps1 create mode 100644 test/Scoop-Download.Tests.ps1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f432537f..1547a91719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,11 @@ ### Bug Fixes -- **decompress**: `Expand-7zipArchive` Only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) +- **decompress**: `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) + +### Code Refactoring + +- **download:** Move download-related functions to 'download.ps1' ([#6095](https://github.com/ScoopInstaller/Scoop/issues/6095)) ## [v0.5.2](https://github.com/ScoopInstaller/Scoop/compare/v0.5.1...v0.5.2) - 2024-07-26 diff --git a/bin/checkhashes.ps1 b/bin/checkhashes.ps1 index cbfb0af40f..6e6420fb2a 100644 --- a/bin/checkhashes.ps1 +++ b/bin/checkhashes.ps1 @@ -46,7 +46,7 @@ param( . "$PSScriptRoot\..\lib\autoupdate.ps1" . "$PSScriptRoot\..\lib\json.ps1" . "$PSScriptRoot\..\lib\versions.ps1" -. "$PSScriptRoot\..\lib\install.ps1" +. "$PSScriptRoot\..\lib\download.ps1" $Dir = Convert-Path $Dir if ($ForceUpdate) { $Update = $true } diff --git a/bin/checkurls.ps1 b/bin/checkurls.ps1 index 64ab0c1840..0ac593362a 100644 --- a/bin/checkurls.ps1 +++ b/bin/checkurls.ps1 @@ -28,7 +28,7 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" -. "$PSScriptRoot\..\lib\install.ps1" +. "$PSScriptRoot\..\lib\download.ps1" $Dir = Convert-Path $Dir $Queue = @() diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index 57010c9992..33a4449488 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -73,7 +73,7 @@ param( . "$PSScriptRoot\..\lib\buckets.ps1" . "$PSScriptRoot\..\lib\json.ps1" . "$PSScriptRoot\..\lib\versions.ps1" -. "$PSScriptRoot\..\lib\install.ps1" # needed for hash generation +. "$PSScriptRoot\..\lib\download.ps1" if ($App -ne '*' -and (Test-Path $App -PathType Leaf)) { $Dir = Split-Path $App diff --git a/bin/describe.ps1 b/bin/describe.ps1 index f9e024e4c4..5faa1c403c 100644 --- a/bin/describe.ps1 +++ b/bin/describe.ps1 @@ -23,6 +23,7 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" . "$PSScriptRoot\..\lib\description.ps1" +. "$PSScriptRoot\..\lib\download.ps1" $Dir = Convert-Path $Dir $Queue = @() diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index bd24e04015..5cef1622a0 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -1,4 +1,17 @@ # Must included with 'json.ps1' + +function format_hash([String] $hash) { + $hash = $hash.toLower() + switch ($hash.Length) { + 32 { $hash = "md5:$hash" } # md5 + 40 { $hash = "sha1:$hash" } # sha1 + 64 { $hash = $hash } # sha256 + 128 { $hash = "sha512:$hash" } # sha512 + default { $hash = $null } + } + return $hash +} + function find_hash_in_rdf([String] $url, [String] $basename) { $xml = $null try { diff --git a/lib/core.ps1 b/lib/core.ps1 index 23a10c32ed..eeaa2c9ad4 100644 --- a/lib/core.ps1 +++ b/lib/core.ps1 @@ -63,18 +63,6 @@ function Optimize-SecurityProtocol { } } -function Get-Encoding($wc) { - if ($null -ne $wc.ResponseHeaders -and $wc.ResponseHeaders['Content-Type'] -match 'charset=([^;]*)') { - return [System.Text.Encoding]::GetEncoding($Matches[1]) - } else { - return [System.Text.Encoding]::GetEncoding('utf-8') - } -} - -function Get-UserAgent() { - return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if(${env:ProgramFiles(Arm)}){'ARM64; '}elseif($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -in 'AMD64','ARM64'){'WOW64; '})$PSEdition)" -} - function Show-DeprecatedWarning { <# .SYNOPSIS @@ -228,35 +216,6 @@ function Complete-ConfigChange { } } -function setup_proxy() { - # note: '@' and ':' in password must be escaped, e.g. 'p@ssword' -> p\@ssword' - $proxy = get_config PROXY - if(!$proxy) { - return - } - try { - $credentials, $address = $proxy -split '(?.*)(?:\/|\?dwl=)(?.*)$") { - $Body = @{ - projectUri = $Matches.name; - fileName = $Matches.filename; - source = 'CF'; - isLatestVersion = $true - } - if ((Invoke-RestMethod -Uri $url) -match '"p":"(?[a-f0-9]{24}).*?"r":"(?[a-f0-9]{24})') { - $Body.Add("projectId", $Matches.pid) - $Body.Add("releaseId", $Matches.rid) - } - $url = Invoke-RestMethod -Method Post -Uri "https://api.fosshub.com/download/" -ContentType "application/json" -Body (ConvertTo-Json $Body -Compress) - if ($null -eq $url.error) { - $url = $url.data.url - } - } - - # Sourceforge.net - if ($url -match "(?:downloads\.)?sourceforge.net\/projects?\/(?[^\/]+)\/(?:files\/)?(?.*?)(?:$|\/download|\?)") { - # Reshapes the URL to avoid redirections - $url = "https://downloads.sourceforge.net/project/$($matches['project'])/$($matches['file'])" - } - - # Github.com - if ($url -match 'github.com/(?[^/]+)/(?[^/]+)/releases/download/(?[^/]+)/(?[^/#]+)(?.*)' -and ($token = Get-GitHubToken)) { - $headers = @{ "Authorization" = "token $token" } - $privateUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)" - $assetUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)/releases/tags/$($Matches.tag)" - - if ((Invoke-RestMethod -Uri $privateUrl -Headers $headers).Private) { - $url = ((Invoke-RestMethod -Uri $assetUrl -Headers $headers).Assets | Where-Object -Property Name -EQ -Value $Matches.file).Url, $Matches.filename -join '' - } - } - - return $url -} - -function get_magic_bytes($file) { - if(!(Test-Path $file)) { - return '' - } - - if((Get-Command Get-Content).parameters.ContainsKey('AsByteStream')) { - # PowerShell Core (6.0+) '-Encoding byte' is replaced by '-AsByteStream' - return Get-Content $file -AsByteStream -TotalCount 8 - } - else { - return Get-Content $file -Encoding byte -TotalCount 8 - } -} - -function get_magic_bytes_pretty($file, $glue = ' ') { - if(!(Test-Path $file)) { - return '' - } - - return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue -} - function Out-UTF8File { param( [Parameter(Mandatory = $True, Position = 0)] @@ -1473,6 +1300,3 @@ $scoopPathEnvVar = switch (get_config USE_ISOLATED_PATH) { # OS information $WindowsBuild = [System.Environment]::OSVersion.Version.Build - -# Setup proxy globally -setup_proxy diff --git a/lib/download.ps1 b/lib/download.ps1 new file mode 100644 index 0000000000..d162fb616e --- /dev/null +++ b/lib/download.ps1 @@ -0,0 +1,758 @@ +# Description: Functions for downloading files + +## Meta downloader + +function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { + # we only want to show this warning once + if (!$use_cache) { warn 'Cache is being ignored.' } + + # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work + $urls = @(script:url $manifest $architecture) + + # can be multiple cookies: they will be used for all HTTP requests. + $cookies = $manifest.cookie + + # download first + if (Test-Aria2Enabled) { + Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash + } else { + foreach ($url in $urls) { + $fname = url_filename $url + + try { + Invoke-CachedDownload $app $version $url "$dir\$fname" $cookies $use_cache + } catch { + Write-Host -ForegroundColor DarkRed $_ + abort "URL $url is not valid" + } + + if ($check_hash) { + $manifest_hash = hash_for_url $manifest $url $architecture + $ok, $err = check_hash "$dir\$fname" $manifest_hash $(show_app $app $bucket) + if (!$ok) { + error $err + $cached = cache_path $app $version $url + if (Test-Path $cached) { + # rm cached file + Remove-Item -Force $cached + } + if ($url.Contains('sourceforge.net')) { + Write-Host -ForegroundColor Yellow 'SourceForge.net is known for causing hash validation fails. Please try again before opening a ticket.' + } + abort $(new_issue_msg $app $bucket 'hash check failed') + } + } + } + } + + return $urls.ForEach({ url_filename $_ }) +} + +## [System.Net] downloader + +function Invoke-CachedDownload ($app, $version, $url, $to, $cookies = $null, $use_cache = $true) { + $cached = cache_path $app $version $url + + if (!(Test-Path $cached) -or !$use_cache) { + ensure $cachedir | Out-Null + Start-Download $url "$cached.download" $cookies + Move-Item "$cached.download" $cached -Force + } else { Write-Host "Loading $(url_remote_filename $url) from cache" } + + if (!($null -eq $to)) { + if ($use_cache) { + Copy-Item $cached $to + } else { + Move-Item $cached $to -Force + } + } +} + +function Start-Download ($url, $to, $cookies) { + $progress = [console]::isoutputredirected -eq $false -and + $host.name -ne 'Windows PowerShell ISE Host' + + try { + $url = handle_special_urls $url + Invoke-Download $url $to $cookies $progress + } catch { + $e = $_.exception + if ($e.Response.StatusCode -eq 'Unauthorized') { + warn 'Token might be misconfigured.' + } + if ($e.innerexception) { $e = $e.innerexception } + throw $e + } +} + +function Invoke-Download ($url, $to, $cookies, $progress) { + # download with filesize and progress indicator + $reqUrl = ($url -split '#')[0] + $wreq = [Net.WebRequest]::Create($reqUrl) + if ($wreq -is [Net.HttpWebRequest]) { + $wreq.UserAgent = Get-UserAgent + if (-not ($url -match 'sourceforge\.net' -or $url -match 'portableapps\.com')) { + $wreq.Referer = strip_filename $url + } + if ($url -match 'api\.github\.com/repos') { + $wreq.Accept = 'application/octet-stream' + $wreq.Headers['Authorization'] = "Bearer $(Get-GitHubToken)" + $wreq.Headers['X-GitHub-Api-Version'] = '2022-11-28' + } + if ($cookies) { + $wreq.Headers.Add('Cookie', (cookie_header $cookies)) + } + + get_config PRIVATE_HOSTS | Where-Object { $_ -ne $null -and $url -match $_.match } | ForEach-Object { + (ConvertFrom-StringData -StringData $_.Headers).GetEnumerator() | ForEach-Object { + $wreq.Headers[$_.Key] = $_.Value + } + } + } + + try { + $wres = $wreq.GetResponse() + } catch [System.Net.WebException] { + $exc = $_.Exception + $handledCodes = @( + [System.Net.HttpStatusCode]::MovedPermanently, # HTTP 301 + [System.Net.HttpStatusCode]::Found, # HTTP 302 + [System.Net.HttpStatusCode]::SeeOther, # HTTP 303 + [System.Net.HttpStatusCode]::TemporaryRedirect # HTTP 307 + ) + + # Only handle redirection codes + $redirectRes = $exc.Response + if ($handledCodes -notcontains $redirectRes.StatusCode) { + throw $exc + } + + # Get the new location of the file + if ((-not $redirectRes.Headers) -or ($redirectRes.Headers -notcontains 'Location')) { + throw $exc + } + + $newUrl = $redirectRes.Headers['Location'] + info "Following redirect to $newUrl..." + + # Handle manual file rename + if ($url -like '*#/*') { + $null, $postfix = $url -split '#/' + $newUrl = "$newUrl#/$postfix" + } + + Invoke-Download $newUrl $to $cookies $progress + return + } + + $total = $wres.ContentLength + if ($total -eq -1 -and $wreq -is [net.ftpwebrequest]) { + $total = ftp_file_size($url) + } + + if ($progress -and ($total -gt 0)) { + [console]::CursorVisible = $false + function Trace-DownloadProgress ($read) { + Write-DownloadProgress $read $total $url + } + } else { + Write-Host "Downloading $url ($(filesize $total))..." + function Trace-DownloadProgress { + #no op + } + } + + try { + $s = $wres.getresponsestream() + $fs = [io.file]::openwrite($to) + $buffer = New-Object byte[] 2048 + $totalRead = 0 + $sw = [diagnostics.stopwatch]::StartNew() + + Trace-DownloadProgress $totalRead + while (($read = $s.read($buffer, 0, $buffer.length)) -gt 0) { + $fs.write($buffer, 0, $read) + $totalRead += $read + if ($sw.elapsedmilliseconds -gt 100) { + $sw.restart() + Trace-DownloadProgress $totalRead + } + } + $sw.stop() + Trace-DownloadProgress $totalRead + } finally { + if ($progress) { + [console]::CursorVisible = $true + Write-Host + } + if ($fs) { + $fs.close() + } + if ($s) { + $s.close() + } + $wres.close() + } +} + +function Format-DownloadProgress ($url, $read, $total, $console) { + $filename = url_remote_filename $url + + # calculate current percentage done + $p = [math]::Round($read / $total * 100, 0) + + # pre-generate LHS and RHS of progress string + # so we know how much space we have + $left = "$filename ($(filesize $total))" + $right = [string]::Format('{0,3}%', $p) + + # calculate remaining width for progress bar + $midwidth = $console.BufferSize.Width - ($left.Length + $right.Length + 8) + + # calculate how many characters are completed + $completed = [math]::Abs([math]::Round(($p / 100) * $midwidth, 0) - 1) + + # generate dashes to symbolise completed + if ($completed -gt 1) { + $dashes = [string]::Join('', ((1..$completed) | ForEach-Object { '=' })) + } + + # this is why we calculate $completed - 1 above + $dashes += switch ($p) { + 100 { '=' } + default { '>' } + } + + # the remaining characters are filled with spaces + $spaces = switch ($dashes.Length) { + $midwidth { [string]::Empty } + default { + [string]::Join('', ((1..($midwidth - $dashes.Length)) | ForEach-Object { ' ' })) + } + } + + "$left [$dashes$spaces] $right" +} + +function Write-DownloadProgress ($read, $total, $url) { + $console = $Host.UI.RawUI + $left = $console.CursorPosition.X + $top = $console.CursorPosition.Y + $width = $console.BufferSize.Width + + if ($read -eq 0) { + $maxOutputLength = $(Format-DownloadProgress $url 100 $total $console).Length + if (($left + $maxOutputLength) -gt $width) { + # not enough room to print progress on this line + # print on new line + Write-Host + $left = 0 + $top = $top + 1 + if ($top -gt $console.CursorPosition.Y) { $top = $console.CursorPosition.Y } + } + } + + Write-Host $(Format-DownloadProgress $url $read $total $console) -NoNewline + [console]::SetCursorPosition($left, $top) +} + +## Aria2 downloader + +function Test-Aria2Enabled { + return (Test-HelperInstalled -Helper Aria2) -and (get_config 'aria2-enabled' $true) +} + +function aria_exit_code($exitcode) { + $codes = @{ + 0 = 'All downloads were successful' + 1 = 'An unknown error occurred' + 2 = 'Timeout' + 3 = 'Resource was not found' + 4 = 'Aria2 saw the specified number of "resource not found" error. See --max-file-not-found option' + 5 = 'Download aborted because download speed was too slow. See --lowest-speed-limit option' + 6 = 'Network problem occurred.' + 7 = 'There were unfinished downloads. This error is only reported if all finished downloads were successful and there were unfinished downloads in a queue when aria2 exited by pressing Ctrl-C by an user or sending TERM or INT signal' + 8 = 'Remote server did not support resume when resume was required to complete download' + 9 = 'There was not enough disk space available' + 10 = 'Piece length was different from one in .aria2 control file. See --allow-piece-length-change option' + 11 = 'Aria2 was downloading same file at that moment' + 12 = 'Aria2 was downloading same info hash torrent at that moment' + 13 = 'File already existed. See --allow-overwrite option' + 14 = 'Renaming file failed. See --auto-file-renaming option' + 15 = 'Aria2 could not open existing file' + 16 = 'Aria2 could not create new file or truncate existing file' + 17 = 'File I/O error occurred' + 18 = 'Aria2 could not create directory' + 19 = 'Name resolution failed' + 20 = 'Aria2 could not parse Metalink document' + 21 = 'FTP command failed' + 22 = 'HTTP response header was bad or unexpected' + 23 = 'Too many redirects occurred' + 24 = 'HTTP authorization failed' + 25 = 'Aria2 could not parse bencoded file (usually ".torrent" file)' + 26 = '".torrent" file was corrupted or missing information that aria2 needed' + 27 = 'Magnet URI was bad' + 28 = 'Bad/unrecognized option was given or unexpected option argument was given' + 29 = 'The remote server was unable to handle the request due to a temporary overloading or maintenance' + 30 = 'Aria2 could not parse JSON-RPC request' + 31 = 'Reserved. Not used' + 32 = 'Checksum validation failed' + } + if ($null -eq $codes[$exitcode]) { + return 'An unknown error occurred' + } + return $codes[$exitcode] +} + +function get_filename_from_metalink($file) { + $bytes = get_magic_bytes_pretty $file '' + # check if file starts with ' p\@ssword' + $proxy = get_config PROXY + if (!$proxy) { + return + } + try { + $credentials, $address = $proxy -split '(?'." + } + $ret +} + +### URL handling + +function handle_special_urls($url) { + # FossHub.com + if ($url -match '^(?:.*fosshub.com\/)(?.*)(?:\/|\?dwl=)(?.*)$') { + $Body = @{ + projectUri = $Matches.name + fileName = $Matches.filename + source = 'CF' + isLatestVersion = $true + } + if ((Invoke-RestMethod -Uri $url) -match '"p":"(?[a-f0-9]{24}).*?"r":"(?[a-f0-9]{24})') { + $Body.Add('projectId', $Matches.pid) + $Body.Add('releaseId', $Matches.rid) + } + $url = Invoke-RestMethod -Method Post -Uri 'https://api.fosshub.com/download/' -ContentType 'application/json' -Body (ConvertTo-Json $Body -Compress) + if ($null -eq $url.error) { + $url = $url.data.url + } + } + + # Sourceforge.net + if ($url -match '(?:downloads\.)?sourceforge.net\/projects?\/(?[^\/]+)\/(?:files\/)?(?.*?)(?:$|\/download|\?)') { + # Reshapes the URL to avoid redirections + $url = "https://downloads.sourceforge.net/project/$($matches['project'])/$($matches['file'])" + } + + # Github.com + if ($url -match 'github.com/(?[^/]+)/(?[^/]+)/releases/download/(?[^/]+)/(?[^/#]+)(?.*)' -and ($token = Get-GitHubToken)) { + $headers = @{ 'Authorization' = "token $token" } + $privateUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)" + $assetUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)/releases/tags/$($Matches.tag)" + + if ((Invoke-RestMethod -Uri $privateUrl -Headers $headers).Private) { + $url = ((Invoke-RestMethod -Uri $assetUrl -Headers $headers).Assets | Where-Object -Property Name -EQ -Value $Matches.file).Url, $Matches.filename -join '' + } + } + + return $url +} + +### Remote file information + +function download_json($url) { + $githubtoken = Get-GitHubToken + $authheader = @{} + if ($githubtoken) { + $authheader = @{'Authorization' = "token $githubtoken" } + } + $ProgressPreference = 'SilentlyContinue' + $result = Invoke-WebRequest $url -UseBasicParsing -Headers $authheader | Select-Object -ExpandProperty content | ConvertFrom-Json + $ProgressPreference = 'Continue' + $result +} + +function get_magic_bytes($file) { + if (!(Test-Path $file)) { + return '' + } + + if ((Get-Command Get-Content).parameters.ContainsKey('AsByteStream')) { + # PowerShell Core (6.0+) '-Encoding byte' is replaced by '-AsByteStream' + return Get-Content $file -AsByteStream -TotalCount 8 + } else { + return Get-Content $file -Encoding byte -TotalCount 8 + } +} + +function get_magic_bytes_pretty($file, $glue = ' ') { + if (!(Test-Path $file)) { + return '' + } + + return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue +} + +Function Get-RemoteFileSize ($Uri) { + $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing + if (!$response.Headers.StatusCode) { + $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } + } +} + +function ftp_file_size($url) { + $request = [net.ftpwebrequest]::create($url) + $request.method = [net.webrequestmethods+ftp]::getfilesize + $request.getresponse().contentlength +} + +function url_filename($url) { + (Split-Path $url -Leaf).split('?') | Select-Object -First 1 +} + +function url_remote_filename($url) { + # Unlike url_filename which can be tricked by appending a + # URL fragment (e.g. #/dl.7z, useful for coercing a local filename), + # this function extracts the original filename from the URL. + $uri = (New-Object URI $url) + $basename = Split-Path $uri.PathAndQuery -Leaf + If ($basename -match '.*[?=]+([\w._-]+)') { + $basename = $matches[1] + } + If (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) { + $basename = Split-Path $uri.AbsolutePath -Leaf + } + If (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) { + $basename = $uri.Fragment.Trim('/', '#') + } + return $basename +} + +### Hash-related functions + +function hash_for_url($manifest, $url, $arch) { + $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } + + if ($hashes.length -eq 0) { return $null } + + $urls = @(script:url $manifest $arch) + + $index = [array]::IndexOf($urls, $url) + if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } + + @($hashes)[$index] +} + +function check_hash($file, $hash, $app_name) { + # returns (ok, err) + if (!$hash) { + warn "Warning: No hash in manifest. SHA256 for '$(fname $file)' is:`n $((Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower())" + return $true, $null + } + + Write-Host 'Checking hash of ' -NoNewline + Write-Host $(url_remote_filename $url) -ForegroundColor Cyan -NoNewline + Write-Host ' ... ' -NoNewline + $algorithm, $expected = get_hash $hash + if ($null -eq $algorithm) { + return $false, "Hash type '$algorithm' isn't supported." + } + + $actual = (Get-FileHash -Path $file -Algorithm $algorithm).Hash.ToLower() + $expected = $expected.ToLower() + + if ($actual -ne $expected) { + $msg = "Hash check failed!`n" + $msg += "App: $app_name`n" + $msg += "URL: $url`n" + if (Test-Path $file) { + $msg += "First bytes: $((get_magic_bytes_pretty $file ' ').ToUpper())`n" + } + if ($expected -or $actual) { + $msg += "Expected: $expected`n" + $msg += "Actual: $actual" + } + return $false, $msg + } + Write-Host 'ok.' -f Green + return $true, $null +} + +function get_hash([String] $multihash) { + $type, $hash = $multihash -split ':' + if (!$hash) { + # no type specified, assume sha256 + $type, $hash = 'sha256', $multihash + } + + if (@('md5', 'sha1', 'sha256', 'sha512') -notcontains $type) { + return $null, "Hash type '$type' isn't supported." + } + + return $type, $hash.ToLower() +} + +# Setup proxy globally +setup_proxy diff --git a/lib/install.ps1 b/lib/install.ps1 index 54b0922e26..56cfdac924 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -81,576 +81,10 @@ function install_app($app, $architecture, $global, $suggested, $use_cache = $tru show_notes $manifest $dir $original_dir $persist_dir } -function Invoke-CachedDownload ($app, $version, $url, $to, $cookies = $null, $use_cache = $true) { - $cached = cache_path $app $version $url - - if (!(Test-Path $cached) -or !$use_cache) { - ensure $cachedir | Out-Null - Start-Download $url "$cached.download" $cookies - Move-Item "$cached.download" $cached -Force - } else { Write-Host "Loading $(url_remote_filename $url) from cache" } - - if (!($null -eq $to)) { - if ($use_cache) { - Copy-Item $cached $to - } else { - Move-Item $cached $to -Force - } - } -} - -function Start-Download ($url, $to, $cookies) { - $progress = [console]::isoutputredirected -eq $false -and - $host.name -ne 'Windows PowerShell ISE Host' - - try { - $url = handle_special_urls $url - Invoke-Download $url $to $cookies $progress - } catch { - $e = $_.exception - if ($e.Response.StatusCode -eq 'Unauthorized') { - warn 'Token might be misconfigured.' - } - if ($e.innerexception) { $e = $e.innerexception } - throw $e - } -} - -function aria_exit_code($exitcode) { - $codes = @{ - 0 = 'All downloads were successful' - 1 = 'An unknown error occurred' - 2 = 'Timeout' - 3 = 'Resource was not found' - 4 = 'Aria2 saw the specified number of "resource not found" error. See --max-file-not-found option' - 5 = 'Download aborted because download speed was too slow. See --lowest-speed-limit option' - 6 = 'Network problem occurred.' - 7 = 'There were unfinished downloads. This error is only reported if all finished downloads were successful and there were unfinished downloads in a queue when aria2 exited by pressing Ctrl-C by an user or sending TERM or INT signal' - 8 = 'Remote server did not support resume when resume was required to complete download' - 9 = 'There was not enough disk space available' - 10 = 'Piece length was different from one in .aria2 control file. See --allow-piece-length-change option' - 11 = 'Aria2 was downloading same file at that moment' - 12 = 'Aria2 was downloading same info hash torrent at that moment' - 13 = 'File already existed. See --allow-overwrite option' - 14 = 'Renaming file failed. See --auto-file-renaming option' - 15 = 'Aria2 could not open existing file' - 16 = 'Aria2 could not create new file or truncate existing file' - 17 = 'File I/O error occurred' - 18 = 'Aria2 could not create directory' - 19 = 'Name resolution failed' - 20 = 'Aria2 could not parse Metalink document' - 21 = 'FTP command failed' - 22 = 'HTTP response header was bad or unexpected' - 23 = 'Too many redirects occurred' - 24 = 'HTTP authorization failed' - 25 = 'Aria2 could not parse bencoded file (usually ".torrent" file)' - 26 = '".torrent" file was corrupted or missing information that aria2 needed' - 27 = 'Magnet URI was bad' - 28 = 'Bad/unrecognized option was given or unexpected option argument was given' - 29 = 'The remote server was unable to handle the request due to a temporary overloading or maintenance' - 30 = 'Aria2 could not parse JSON-RPC request' - 31 = 'Reserved. Not used' - 32 = 'Checksum validation failed' - } - if ($null -eq $codes[$exitcode]) { - return 'An unknown error occurred' - } - return $codes[$exitcode] -} - -function get_filename_from_metalink($file) { - $bytes = get_magic_bytes_pretty $file '' - # check if file starts with '' } - } - - # the remaining characters are filled with spaces - $spaces = switch ($dashes.Length) { - $midwidth { [string]::Empty } - default { - [string]::Join('', ((1..($midwidth - $dashes.Length)) | ForEach-Object { ' ' })) - } - } - - "$left [$dashes$spaces] $right" -} - -function Write-DownloadProgress ($read, $total, $url) { - $console = $host.UI.RawUI - $left = $console.CursorPosition.X - $top = $console.CursorPosition.Y - $width = $console.BufferSize.Width - - if ($read -eq 0) { - $maxOutputLength = $(Format-DownloadProgress $url 100 $total $console).length - if (($left + $maxOutputLength) -gt $width) { - # not enough room to print progress on this line - # print on new line - Write-Host - $left = 0 - $top = $top + 1 - if ($top -gt $console.CursorPosition.Y) { $top = $console.CursorPosition.Y } - } - } - - Write-Host $(Format-DownloadProgress $url $read $total $console) -NoNewline - [console]::SetCursorPosition($left, $top) -} - -function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { - # we only want to show this warning once - if (!$use_cache) { warn 'Cache is being ignored.' } - - # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work - $urls = @(script:url $manifest $architecture) - - # can be multiple cookies: they will be used for all HTTP requests. - $cookies = $manifest.cookie - - # download first - if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash - } else { - foreach ($url in $urls) { - $fname = url_filename $url - - try { - Invoke-CachedDownload $app $version $url "$dir\$fname" $cookies $use_cache - } catch { - Write-Host -f darkred $_ - abort "URL $url is not valid" - } - - if ($check_hash) { - $manifest_hash = hash_for_url $manifest $url $architecture - $ok, $err = check_hash "$dir\$fname" $manifest_hash $(show_app $app $bucket) - if (!$ok) { - error $err - $cached = cache_path $app $version $url - if (Test-Path $cached) { - # rm cached file - Remove-Item -Force $cached - } - if ($url.Contains('sourceforge.net')) { - Write-Host -f yellow 'SourceForge.net is known for causing hash validation fails. Please try again before opening a ticket.' - } - abort $(new_issue_msg $app $bucket 'hash check failed') - } - } - } - } - - return $urls.ForEach({ url_filename $_ }) -} - -function cookie_header($cookies) { - if (!$cookies) { return } - - $vals = $cookies.psobject.properties | ForEach-Object { - "$($_.name)=$($_.value)" - } - - [string]::join(';', $vals) -} - function is_in_dir($dir, $check) { $check -match "^$([regex]::Escape("$dir"))([/\\]|$)" } -function ftp_file_size($url) { - $request = [net.ftpwebrequest]::create($url) - $request.method = [net.webrequestmethods+ftp]::getfilesize - $request.getresponse().contentlength -} - -# hashes -function hash_for_url($manifest, $url, $arch) { - $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } - - if ($hashes.length -eq 0) { return $null } - - $urls = @(script:url $manifest $arch) - - $index = [array]::indexof($urls, $url) - if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } - - @($hashes)[$index] -} - -# returns (ok, err) -function check_hash($file, $hash, $app_name) { - if (!$hash) { - warn "Warning: No hash in manifest. SHA256 for '$(fname $file)' is:`n $((Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower())" - return $true, $null - } - - Write-Host 'Checking hash of ' -NoNewline - Write-Host $(url_remote_filename $url) -f Cyan -NoNewline - Write-Host ' ... ' -NoNewline - $algorithm, $expected = get_hash $hash - if ($null -eq $algorithm) { - return $false, "Hash type '$algorithm' isn't supported." - } - - $actual = (Get-FileHash -Path $file -Algorithm $algorithm).Hash.ToLower() - $expected = $expected.ToLower() - - if ($actual -ne $expected) { - $msg = "Hash check failed!`n" - $msg += "App: $app_name`n" - $msg += "URL: $url`n" - if (Test-Path $file) { - $msg += "First bytes: $((get_magic_bytes_pretty $file ' ').ToUpper())`n" - } - if ($expected -or $actual) { - $msg += "Expected: $expected`n" - $msg += "Actual: $actual" - } - return $false, $msg - } - Write-Host 'ok.' -f Green - return $true, $null -} - function Invoke-Installer { [CmdletBinding()] param ( diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index 1901527b09..f47a672e5a 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -23,7 +23,7 @@ . "$PSScriptRoot\..\lib\json.ps1" # 'autoupdate.ps1' (indirectly) . "$PSScriptRoot\..\lib\autoupdate.ps1" # 'generate_user_manifest' (indirectly) . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' -. "$PSScriptRoot\..\lib\install.ps1" +. "$PSScriptRoot\..\lib\download.ps1" if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } diff --git a/libexec/scoop-info.ps1 b/libexec/scoop-info.ps1 index 3907b9f71a..7b23544551 100644 --- a/libexec/scoop-info.ps1 +++ b/libexec/scoop-info.ps1 @@ -6,6 +6,7 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' . "$PSScriptRoot\..\lib\versions.ps1" # 'Get-InstalledVersion' +. "$PSScriptRoot\..\lib\download.ps1" # 'Get-RemoteFileSize' $opt, $app, $err = getopt $args 'v' 'verbose' if ($err) { error "scoop info: $err"; exit 1 } @@ -166,7 +167,7 @@ if ($status.installed) { $cached = $null } - $urlLength = (Invoke-WebRequest $url -Method Head).Headers.'Content-Length' | ForEach-Object { [int]$_ } + $urlLength = Get-RemoteFileSize $url $totalPackage += $urlLength } catch [System.Management.Automation.RuntimeException] { $totalPackage = 0 diff --git a/libexec/scoop-install.ps1 b/libexec/scoop-install.ps1 index b7dda05d3c..0fadcff09d 100644 --- a/libexec/scoop-install.ps1 +++ b/libexec/scoop-install.ps1 @@ -33,6 +33,7 @@ . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' 'Select-CurrentVersion' (indirectly) . "$PSScriptRoot\..\lib\system.ps1" . "$PSScriptRoot\..\lib\install.ps1" +. "$PSScriptRoot\..\lib\download.ps1" . "$PSScriptRoot\..\lib\decompress.ps1" . "$PSScriptRoot\..\lib\shortcuts.ps1" . "$PSScriptRoot\..\lib\psmodules.ps1" diff --git a/libexec/scoop-search.ps1 b/libexec/scoop-search.ps1 index 4254099989..d94b18660d 100644 --- a/libexec/scoop-search.ps1 +++ b/libexec/scoop-search.ps1 @@ -10,15 +10,10 @@ param($query) . "$PSScriptRoot\..\lib\manifest.ps1" # 'manifest' . "$PSScriptRoot\..\lib\versions.ps1" # 'Get-LatestVersion' +. "$PSScriptRoot\..\lib\download.ps1" $list = [System.Collections.Generic.List[PSCustomObject]]::new() -$githubtoken = Get-GitHubToken -$authheader = @{} -if ($githubtoken) { - $authheader = @{'Authorization' = "token $githubtoken" } -} - function bin_match($manifest, $query) { if (!$manifest.bin) { return $false } $bins = foreach ($bin in $manifest.bin) { @@ -122,23 +117,6 @@ function search_bucket_legacy($bucket, $query) { } } -function download_json($url) { - $ProgressPreference = 'SilentlyContinue' - $result = Invoke-WebRequest $url -UseBasicParsing -Headers $authheader | Select-Object -ExpandProperty content | ConvertFrom-Json - $ProgressPreference = 'Continue' - $result -} - -function github_ratelimit_reached { - $api_link = 'https://api.github.com/rate_limit' - $ret = (download_json $api_link).rate.remaining -eq 0 - if ($ret) { - Write-Host "GitHub API rate limit reached. -Please try again later or configure your API token using 'scoop config gh_token '." - } - $ret -} - function search_remote($bucket, $query) { $uri = [System.Uri](known_bucket_repo $bucket) if ($uri.AbsolutePath -match '/([a-zA-Z0-9]*)/([a-zA-Z0-9-]*)(?:.git|/)?') { diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index bd825731c3..9d41906eca 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -24,6 +24,7 @@ . "$PSScriptRoot\..\lib\versions.ps1" . "$PSScriptRoot\..\lib\depends.ps1" . "$PSScriptRoot\..\lib\install.ps1" +. "$PSScriptRoot\..\lib\download.ps1" if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 0782c7e225..72f1d2f14f 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -31,7 +31,7 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' . "$PSScriptRoot\..\lib\json.ps1" # 'json_path' -. "$PSScriptRoot\..\lib\install.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\download.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' $opt, $apps, $err = getopt $args 'asnup' @('all', 'scan', 'no-depends', 'no-update-scoop', 'passthru') @@ -86,11 +86,6 @@ Function ConvertTo-VirusTotalUrlId ($url) { $url_id } -Function Get-RemoteFileSize ($url) { - $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing - $response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) } -} - Function Get-VirusTotalResultByHash ($hash, $url, $app) { $hash = $hash.ToLower() $api_url = "https://www.virustotal.com/api/v3/files/$hash" diff --git a/test/Scoop-Core.Tests.ps1 b/test/Scoop-Core.Tests.ps1 index f6846e5456..d1cc9defe3 100644 --- a/test/Scoop-Core.Tests.ps1 +++ b/test/Scoop-Core.Tests.ps1 @@ -73,28 +73,6 @@ Describe 'Test-HelperInstalled' -Tag 'Scoop' { } } -Describe 'Test-Aria2Enabled' -Tag 'Scoop' { - It 'should return true if aria2 is installed' { - Mock Test-HelperInstalled { $true } - Mock get_config { $true } - Test-Aria2Enabled | Should -BeTrue - } - - It 'should return false if aria2 is not installed' { - Mock Test-HelperInstalled { $false } - Mock get_config { $false } - Test-Aria2Enabled | Should -BeFalse - - Mock Test-HelperInstalled { $false } - Mock get_config { $true } - Test-Aria2Enabled | Should -BeFalse - - Mock Test-HelperInstalled { $true } - Mock get_config { $false } - Test-Aria2Enabled | Should -BeFalse - } -} - Describe 'Test-CommandAvailable' -Tag 'Scoop' { It 'should return true if command exists' { Test-CommandAvailable 'Write-Host' | Should -BeTrue diff --git a/test/Scoop-Download.Tests.ps1 b/test/Scoop-Download.Tests.ps1 new file mode 100644 index 0000000000..8968c30014 --- /dev/null +++ b/test/Scoop-Download.Tests.ps1 @@ -0,0 +1,49 @@ +BeforeAll { + . "$PSScriptRoot\Scoop-TestLib.ps1" + . "$PSScriptRoot\..\lib\core.ps1" + . "$PSScriptRoot\..\lib\download.ps1" +} + +Describe 'Test-Aria2Enabled' -Tag 'Scoop' { + It 'should return true if aria2 is installed' { + Mock Test-HelperInstalled { $true } + Mock get_config { $true } + Test-Aria2Enabled | Should -BeTrue + } + + It 'should return false if aria2 is not installed' { + Mock Test-HelperInstalled { $false } + Mock get_config { $false } + Test-Aria2Enabled | Should -BeFalse + + Mock Test-HelperInstalled { $false } + Mock get_config { $true } + Test-Aria2Enabled | Should -BeFalse + + Mock Test-HelperInstalled { $true } + Mock get_config { $false } + Test-Aria2Enabled | Should -BeFalse + } +} + +Describe 'url_filename' -Tag 'Scoop' { + It 'should extract the real filename from an url' { + url_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' + url_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' + } + + It 'can be tricked with a hash to override the real filename' { + url_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo.zip' + } +} + +Describe 'url_remote_filename' -Tag 'Scoop' { + It 'should extract the real filename from an url' { + url_remote_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' + url_remote_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' + } + + It 'can not be tricked with a hash to override the real filename' { + url_remote_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo-v2.zip' + } +} diff --git a/test/Scoop-Install.Tests.ps1 b/test/Scoop-Install.Tests.ps1 index 966c2c0c50..3e564d1964 100644 --- a/test/Scoop-Install.Tests.ps1 +++ b/test/Scoop-Install.Tests.ps1 @@ -12,28 +12,6 @@ Describe 'appname_from_url' -Tag 'Scoop' { } } -Describe 'url_filename' -Tag 'Scoop' { - It 'should extract the real filename from an url' { - url_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' - url_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' - } - - It 'can be tricked with a hash to override the real filename' { - url_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo.zip' - } -} - -Describe 'url_remote_filename' -Tag 'Scoop' { - It 'should extract the real filename from an url' { - url_remote_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' - url_remote_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' - } - - It 'can not be tricked with a hash to override the real filename' { - url_remote_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo-v2.zip' - } -} - Describe 'is_in_dir' -Tag 'Scoop', 'Windows' { It 'should work correctly' { is_in_dir 'C:\test' 'C:\foo' | Should -BeFalse From e0c682de7c1f36ec8ea17b3f9eea27184d398854 Mon Sep 17 00:00:00 2001 From: kiennq Date: Mon, 26 Aug 2024 01:10:30 -0700 Subject: [PATCH 5/8] fix(download): Fallback to default downloader when aria2 fails (#4292) --- CHANGELOG.md | 1 + lib/download.ps1 | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1547a91719..76c04c961d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Bug Fixes +- **scoop-download|install|update:** Fallback to default downloader when aria2 fails ([#4292](https://github.com/ScoopInstaller/Scoop/issues/4292)) - **decompress**: `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) ### Code Refactoring diff --git a/lib/download.ps1 b/lib/download.ps1 index d162fb616e..70641cca7f 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -138,7 +138,7 @@ function Invoke-Download ($url, $to, $cookies, $progress) { # Handle manual file rename if ($url -like '*#/*') { $null, $postfix = $url -split '#/' - $newUrl = "$newUrl#/$postfix" + $newUrl = "$newUrl`#/$postfix" } Invoke-Download $newUrl $to $cookies $progress @@ -454,10 +454,21 @@ function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $ Write-Host '' if ($lastexitcode -gt 0) { - error "Download failed! (Error $lastexitcode) $(aria_exit_code $lastexitcode)" - error $urlstxt_content - error $aria2 - abort $(new_issue_msg $app $bucket 'download via aria2 failed') + warn "Download failed! (Error $lastexitcode) $(aria_exit_code $lastexitcode)" + warn $urlstxt_content + warn $aria2 + warn $(new_issue_msg $app $bucket "download via aria2 failed") + + Write-Host "Fallback to default downloader ..." + + try { + foreach ($url in $urls) { + Invoke-CachedDownload $app $version $url "$($data.$url.target)" $cookies $use_cache + } + } catch { + Write-Host $_ -ForegroundColor DarkRed + abort "URL $url is not valid" + } } # remove aria2 input file when done From 3577f91d82082f2a6e5467539c319ab4e11fde0c Mon Sep 17 00:00:00 2001 From: HUMORCE Date: Fri, 20 Sep 2024 14:18:13 +0800 Subject: [PATCH 6/8] fix(commands): Handling broken aliases (#6141) --- CHANGELOG.md | 1 + lib/commands.ps1 | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c04c961d..356486b519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - **scoop-download|install|update:** Fallback to default downloader when aria2 fails ([#4292](https://github.com/ScoopInstaller/Scoop/issues/4292)) - **decompress**: `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) +- **commands**: Handling broken aliases ([#6141](https://github.com/ScoopInstaller/Scoop/issues/6141)) ### Code Refactoring diff --git a/lib/commands.ps1 b/lib/commands.ps1 index 04775de357..6812aebd52 100644 --- a/lib/commands.ps1 +++ b/lib/commands.ps1 @@ -4,7 +4,7 @@ function command_files { (Get-ChildItem "$PSScriptRoot\..\libexec") + (Get-ChildItem "$scoopdir\shims") | - Where-Object 'scoop-.*?\.ps1$' -Property Name -Match + Where-Object 'scoop-.*?\.ps1$' -Property Name -Match } function commands { @@ -86,7 +86,9 @@ function rm_alias { } info "Removing alias '$name'..." - Remove-Item "$(shimdir $false)\scoop-$name.ps1" + if (Test-Path "$(shimdir $false)\scoop-$name.ps1") { + Remove-Item "$(shimdir $false)\scoop-$name.ps1" + } $aliases.PSObject.Properties.Remove($name) set_config ALIAS $aliases | Out-Null } @@ -98,11 +100,19 @@ function list_aliases { $aliases = get_config ALIAS ([PSCustomObject]@{}) $alias_info = $aliases.PSObject.Properties.Name | Where-Object { $_ } | ForEach-Object { + # Mark the alias as , if the alias script file does NOT exist. + if (!(Test-Path "$(shimdir $false)\scoop-$_.ps1")) { + [PSCustomObject]@{ + Name = $_ + Command = '' + } + return + } $content = Get-Content (command_path $_) [PSCustomObject]@{ Name = $_ - Summary = (summary $content).Trim() Command = ($content | Select-Object -Skip 1).Trim() + Summary = (summary $content).Trim() } } if (!$alias_info) { From 7a309a1b003a583244aa6e08ab6a9cfe66228e17 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 9 Oct 2024 19:10:42 +0800 Subject: [PATCH 7/8] fix(shim): properly check `wslpath`/`cygpath` command first (#6114) Co-authored-by: Hsiao-nan Cheung --- CHANGELOG.md | 1 + lib/core.ps1 | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 356486b519..bdfc332dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - **scoop-download|install|update:** Fallback to default downloader when aria2 fails ([#4292](https://github.com/ScoopInstaller/Scoop/issues/4292)) - **decompress**: `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) - **commands**: Handling broken aliases ([#6141](https://github.com/ScoopInstaller/Scoop/issues/6141)) +- **shim:** Do not suppress `stderr`, properly check `wslpath`/`cygpath` command first ([#6114](https://github.com/ScoopInstaller/Scoop/pull/6114)) ### Code Refactoring diff --git a/lib/core.ps1 b/lib/core.ps1 index eeaa2c9ad4..c271ce8f94 100644 --- a/lib/core.ps1 +++ b/lib/core.ps1 @@ -1003,11 +1003,18 @@ function shim($path, $global, $name, $arg) { ) -join "`n" | Out-UTF8File $shim -NoNewLine } else { warn_on_overwrite "$shim.cmd" $path + $quoted_arg = if ($arg.Count -gt 0) { $arg | ForEach-Object { "`"$_`"" } } @( "@rem $resolved_path", - "@bash `"`$(wslpath -u '$resolved_path')`" $arg %* 2>nul", - '@if %errorlevel% neq 0 (', - " @bash `"`$(cygpath -u '$resolved_path')`" $arg %* 2>nul", + '@echo off', + 'bash -c "command -v wslpath >/dev/null"', + 'if %errorlevel% equ 0 (', + " bash `"`$(wslpath -u '$resolved_path')`" $quoted_arg %*", + ') else (', + " set args=$quoted_arg %*", + ' setlocal enabledelayedexpansion', + ' if not "!args!"=="" set args=!args:"=""!', + " bash -c `"`$(cygpath -u '$resolved_path') !args!`"", ')' ) -join "`r`n" | Out-UTF8File "$shim.cmd" From 84e00fdb77f84389c1b62b9715a20faa77ef3a94 Mon Sep 17 00:00:00 2001 From: Chawye Hsu Date: Fri, 11 Oct 2024 14:20:43 +0800 Subject: [PATCH 8/8] fix(scoop-bucket): Add missing import for `no_junction` envs (#6181) Signed-off-by: Chawye Hsu --- CHANGELOG.md | 5 +++-- libexec/scoop-bucket.ps1 | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdfc332dac..d9a56e46ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ - **scoop-download|install|update:** Fallback to default downloader when aria2 fails ([#4292](https://github.com/ScoopInstaller/Scoop/issues/4292)) - **decompress**: `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) - **commands**: Handling broken aliases ([#6141](https://github.com/ScoopInstaller/Scoop/issues/6141)) -- **shim:** Do not suppress `stderr`, properly check `wslpath`/`cygpath` command first ([#6114](https://github.com/ScoopInstaller/Scoop/pull/6114)) +- **shim:** Do not suppress `stderr`, properly check `wslpath`/`cygpath` command first ([#6114](https://github.com/ScoopInstaller/Scoop/issues/6114)) +- **scoop-bucket:** Add missing import for `no_junction` envs ([#6181](https://github.com/ScoopInstaller/Scoop/issues/6181)) ### Code Refactoring @@ -53,7 +54,7 @@ - **checkver:** Correct error messages ([#6024](https://github.com/ScoopInstaller/Scoop/issues/6024)) - **core:** Search for Git executable instead of any cmdlet ([#5998](https://github.com/ScoopInstaller/Scoop/issues/5998)) - **core:** Use correct path in 'bash' ([#6006](https://github.com/ScoopInstaller/Scoop/issues/6006)) -- **core:** Limit the number of commands to get when search for git executable ([#6013](https://github.com/ScoopInstaller/Scoop/pull/6013)) +- **core:** Limit the number of commands to get when search for git executable ([#6013](https://github.com/ScoopInstaller/Scoop/issues/6013)) - **decompress:** Match `extract_dir`/`extract_to` and archives ([#5983](https://github.com/ScoopInstaller/Scoop/issues/5983)) - **json:** Serialize jsonpath return ([#5921](https://github.com/ScoopInstaller/Scoop/issues/5921)) - **shim:** Restore original path for JAR cmd ([#6030](https://github.com/ScoopInstaller/Scoop/issues/6030)) diff --git a/libexec/scoop-bucket.ps1 b/libexec/scoop-bucket.ps1 index ceb28865a5..d3b7728559 100644 --- a/libexec/scoop-bucket.ps1 +++ b/libexec/scoop-bucket.ps1 @@ -19,6 +19,10 @@ # scoop bucket known param($cmd, $name, $repo) +if (get_config NO_JUNCTION) { + . "$PSScriptRoot\..\lib\versions.ps1" +} + if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\manifest.ps1" . "$PSScriptRoot\..\lib\database.ps1"