Support files without extensions in importFromUrl (#1873)

* Enhance URL import to work files without extension
* Improve unknown file type when importing from URL
* Update app/Image/Files/BaseMediaFile.php
* minor fix, use quick returns instead of large if-else

app/Actions/Import/FromUrl.php

@@ -55,9 +55,11 @@ public function do(array $urls, ?Album $album, int $intendedOwnerId): Collection
 				$path = parse_url($url, PHP_URL_PATH);
 				$extension = '.' . pathinfo($path, PATHINFO_EXTENSION);
 
-				// Validate photo extension even when `$create->add()` will do later.
-				// This prevents us from downloading unsupported files.
-				BaseMediaFile::assertIsSupportedOrAcceptedFileExtension($extension);
+				if ($extension !== '.') {
+					// Validate photo extension even when `$create->add()` will do later.
+					// This prevents us from downloading unsupported files.
+					BaseMediaFile::assertIsSupportedOrAcceptedFileExtension($extension);
+				}
 
 				// Download file
 				$downloadedFile = new DownloadedFile($url);

app/Image/Files/BaseMediaFile.php

@@ -76,6 +76,24 @@ abstract class BaseMediaFile extends AbstractBinaryBlob implements MediaFile
 		'application/octet-stream', // Some mp4 files; will be corrected by the metadata extractor
 	];
 
+	public const MIME_TYPES_TO_FILE_EXTENSIONS = [
+		'image/gif' => '.gif',
+		'image/jpeg' => '.jpg',
+		'image/png' => '.png',
+		'image/webp' => '.webp',
+		'video/mp4' => '.mp4',
+		'video/mpeg' => '.mpg',
+		'image/x-tga' => '.mpg',
+		'video/ogg' => '.ogv',
+		'video/webm' => '.webm',
+		'video/quicktime' => '.mov',
+		'video/x-ms-asf' => '.wmv',
+		'video/x-ms-wmv' => '.wmv',
+		'video/x-msvideo' => '.avi',
+		'video/x-m4v' => '.avi',
+		'application/octet-stream' => '.mp4',
+	];
+
 	/** @var string[] the accepted raw file extensions minus supported extensions */
 	private static array $cachedAcceptedRawFileExtensions = [];
 
@@ -326,4 +344,30 @@ public static function assertIsSupportedOrAcceptedFileExtension(string $extensio
 			throw new MediaFileUnsupportedException(MediaFileUnsupportedException::DEFAULT_MESSAGE . ' (bad extension: ' . $extension . ')');
 		}
 	}
+
+	/**
+	 * Check if the given mimetype is supported or accepted.
+	 *
+	 * @param ?string $mimeType the file mimetype
+	 *
+	 * @return bool
+	 */
+	public static function isSupportedMimeType(?string $mimeType): bool
+	{
+		return
+			self::isSupportedImageMimeType($mimeType) ||
+			self::isSupportedVideoMimeType($mimeType);
+	}
+
+	/**
+	 * Returns the default file extension for the given MIME type or an empty string if there is no default extension.
+	 *
+	 * @param string $mimeType a MIME type
+	 *
+	 * @return string the default file extension for the given MIME type
+	 */
+	public static function getDefaultFileExtensionForMimeType(string $mimeType): string
+	{
+		return self::MIME_TYPES_TO_FILE_EXTENSIONS[strtolower($mimeType)] ?? '';
+	}
 }

app/Image/Files/DownloadedFile.php

@@ -3,11 +3,16 @@
 namespace App\Image\Files;
 
 use App\Exceptions\MediaFileOperationException;
+use App\Exceptions\MediaFileUnsupportedException;
 use Safe\Exceptions\PcreException;
 use function Safe\fclose;
 use function Safe\fopen;
+use function Safe\mime_content_type;
 use function Safe\parse_url;
 use function Safe\preg_match;
+use function Safe\rewind;
+use function Safe\stream_copy_to_stream;
+use function Safe\tmpfile;
 
 /**
  * Represents a temporary local file which has been downloaded.
@@ -31,10 +36,12 @@ public function __construct(string $url)
 			$path = parse_url($url, PHP_URL_PATH);
 			$basename = pathinfo($path, PATHINFO_FILENAME);
 			$extension = '.' . pathinfo($path, PATHINFO_EXTENSION);
-			parent::__construct($extension, $basename);
 
 			$downloadStream = fopen($url, 'rb');
 			$downloadStreamData = stream_get_meta_data($downloadStream);
+
+			/** @var string|null $originalMimeType */
+			$originalMimeType = null;
 			// Find the server-side MIME type; the HTTP headers are part of
 			// the protocol-specific meta-data of the stream handler
 			foreach ($downloadStreamData['wrapper_data'] as $http_header) {
@@ -46,12 +53,55 @@ public function __construct(string $url)
 					PREG_UNMATCHED_AS_NULL
 				);
 				if (count($matches) === 2 && $matches[1]) {
-					$this->originalMimeType = $matches[1];
+					$originalMimeType = $matches[1];
 					break;
 				}
 			}
-			$this->write($downloadStream);
+
+			// When the URL doesn't contain the file's extension, the web server may or may have not set the
+			// Content-Type correctly. If the Content-Type header has a value that we recognize, we consider it valid.
+			// In all other cases we try to guess the file type.
+			// File extension > Content-Type > Inferred MIME type
+
+			if (self::isSupportedOrAcceptedFileExtension($extension)) {
+				parent::__construct($extension, $basename);
+				$this->originalMimeType = $originalMimeType;
+				$this->write($downloadStream);
+				fclose($downloadStream);
+
+				return;
+			}
+
+			if (self::isSupportedMimeType($originalMimeType)) {
+				$extension = self::getDefaultFileExtensionForMimeType($originalMimeType);
+				parent::__construct($extension, $basename);
+				$this->originalMimeType = $originalMimeType;
+				$this->write($downloadStream);
+				fclose($downloadStream);
+
+				return;
+			}
+
+			$temp = tmpfile();
+			stream_copy_to_stream($downloadStream, $temp);
 			fclose($downloadStream);
+
+			rewind($temp);
+			$originalMimeType = mime_content_type($temp);
+
+			if (self::isSupportedMimeType($originalMimeType)) {
+				$extension = self::getDefaultFileExtensionForMimeType($originalMimeType);
+				parent::__construct($extension, $basename);
+				$this->originalMimeType = $originalMimeType;
+				rewind($temp);
+				$this->write($temp);
+				fclose($temp);
+
+				return;
+			}
+
+			fclose($temp);
+			throw new MediaFileUnsupportedException(MediaFileUnsupportedException::DEFAULT_MESSAGE . ' (bad file type: ' . $originalMimeType . ')');
 		} catch (\ErrorException|PcreException $e) {
 			throw new MediaFileOperationException($e->getMessage(), $e);
 		}

tests/Feature/Constants/TestConstants.php

@@ -17,7 +17,9 @@ class TestConstants
 	public const MIME_TYPE_VID_QUICKTIME = 'video/quicktime';
 
 	public const SAMPLE_DOWNLOAD_JPG = 'https://github.com/LycheeOrg/Lychee/raw/master/tests/Samples/mongolia.jpeg';
+	public const SAMPLE_DOWNLOAD_JPG_WITHOUT_EXTENSION = 'https://github.com/LycheeOrg/Lychee/raw/master/tests/Samples/mongolia';
 	public const SAMPLE_DOWNLOAD_TIFF = 'https://github.com/LycheeOrg/Lychee/raw/master/tests/Samples/tiff.tif';
+	public const SAMPLE_DOWNLOAD_TIFF_WITHOUT_EXTENSION = 'https://github.com/wladif/Lychee/raw/master/tests/Samples/tiff';
 
 	public const SAMPLE_FILE_AARHUS = 'tests/Samples/aarhus.jpg';
 	public const SAMPLE_FILE_ETTLINGEN = 'tests/Samples/ettlinger-alb.jpg';

tests/Feature/PhotosAddMethodsTest.php

@@ -214,6 +214,24 @@ public function testImportFromUrl(): void
 		]]);
 	}
 
+	public function testImportFromUrlWithoutExtension(): void
+	{
+		$response = $this->photos_tests->importFromUrl([TestConstants::SAMPLE_DOWNLOAD_JPG_WITHOUT_EXTENSION]);
+
+		$response->assertJson([[
+			'album_id' => null,
+			'title' => 'mongolia',
+			'type' => TestConstants::MIME_TYPE_IMG_JPEG,
+			'size_variants' => [
+				'original' => [
+					'width' => 1280,
+					'height' => 850,
+					'filesize' => 201316,
+				],
+			],
+		]]);
+	}
+
 	/**
 	 * Test import from URL of a supported raw image.
 	 *

tests/Feature/PhotosAddNegativeTest.php

@@ -127,6 +127,31 @@ public function testRefusedRawImportFormUrl(): void
 		}
 	}
 
+	/**
+	 * Test import from URL of an unsupported raw image without file extension.
+	 *
+	 * We need this test because in case the file doesn't have an extension, we'll download the file
+	 * and try to guess the extension.
+	 *
+	 * @return void
+	 */
+	public function testRefusedRawImportFormUrlWithoutExtension(): void
+	{
+		$acceptedRawFormats = static::getAcceptedRawFormats();
+		try {
+			static::setAcceptedRawFormats('');
+
+			$this->photos_tests->importFromUrl(
+				[TestConstants::SAMPLE_DOWNLOAD_TIFF_WITHOUT_EXTENSION],
+				null,
+				422,
+				'MediaFileUnsupportedException'
+			);
+		} finally {
+			static::setAcceptedRawFormats($acceptedRawFormats);
+		}
+	}
+
 	/**
 	 * Recursively restricts the access to the given directory.
 	 *