From 60494af69367d4108e3cd6508049f474a2b8fe54 Mon Sep 17 00:00:00 2001 From: DeadSix27 Date: Sun, 29 Dec 2019 15:38:23 +0100 Subject: [PATCH] initial commit --- .gitattributes | 2 + README.md | 11 + libs/pathlibex/README.md | 0 libs/pathlibex/pathlibex.py | 236 ++++++++++++++++++ tools/opus_maker/README.md | 0 tools/opus_maker/opus.py | 478 ++++++++++++++++++++++++++++++++++++ 6 files changed, 727 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 libs/pathlibex/README.md create mode 100644 libs/pathlibex/pathlibex.py create mode 100644 tools/opus_maker/README.md create mode 100644 tools/opus_maker/opus.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..09bc62c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2ffac10 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Random collection of Python stuff + +#### Some of it might helpful to others. + +# Tools +- [opus.py](https://github.com/DeadSix27/various_python_tools/tree/master/tools/opus_maker) + Tool to encode and share opus audio files. + +# Libraries +- [pathlibex.py](https://github.com/DeadSix27/various_python_tools/tree/master/libs/pathlibex) + Basic extension to the pathlib (mostly used internally) diff --git a/libs/pathlibex/README.md b/libs/pathlibex/README.md new file mode 100644 index 0000000..e69de29 diff --git a/libs/pathlibex/pathlibex.py b/libs/pathlibex/pathlibex.py new file mode 100644 index 0000000..48a5174 --- /dev/null +++ b/libs/pathlibex/pathlibex.py @@ -0,0 +1,236 @@ +# Copyright (C) 2019 +# +# This source code is licensed under a +# Creative Commons Attribution-NonCommercial 4.0 International License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# You should have received a copy of the license along with this +# work. If not, see . + +from __future__ import annotations + +import os +import pathlib +import re +import shutil +from typing import AnyStr, List, Optional, Tuple, Union + +import magic + + +class Path(pathlib.Path): + ''' + Very crude and simple pathlib extension, + mostly used internally for my needs, so might be buggy and code is unclean. + ''' + _flavour = pathlib._windows_flavour if os.name == 'nt' else pathlib._posix_flavour + + def __new__(cls, *args): + return super(Path, cls).__new__(cls, *args) + + def __init__(self, *args): + super().__init__() + self.ssuffix = self.suffix.lstrip(".") + self._some_instance_ppath_value = self.exists() + + def listfiles(self, extensions=()) -> List[Path]: + '''### listfiles + ##### listfiles + + ### Args: + `extensions` (tuple, optional): List of extensions to limit listing to, with dot prefix. Defaults to (). + + ### Returns: + List[Path]: List of Paths, matching the optionally specificed extension(s) + ''' + lst = None + if len(extensions) > 0: + lst = [self.joinpath(x) for x in self._accessor.listdir(self) if self.joinpath(x).is_file() and x.lower().endswith(extensions)] + else: + lst = [self.joinpath(x) for x in self._accessor.listdir(self) if self.joinpath(x).is_file()] + + def convert(text): + return int(text) if text.isdigit() else text + + def alphanum_key(key): + return [convert(c) for c in re.split('([0-9]+)', str(key))] + + lst = sorted(lst, key=alphanum_key) + return lst + + def listall(self, recursive=False) -> List[Path]: + + lst = [] + if all: + for r, dirs, files in os.walk(self): + for f in files: + lst.append(Path(os.path.join(r, f))) + else: + lst = [self.joinpath(x) for x in self._accessor.listdir(self)] + + def convert(text): + return int(text) if text.isdigit() else text + + def alphanum_key(key): + return [convert(c) for c in re.split('([0-9]+)', str(key))] + + lst = sorted(lst, key=alphanum_key) + return lst + + def listdirs(self) -> List[Path]: + '''### listdirs + ##### Same as listfiles, except for directories only. + + ### Returns: + List[Path]: List of Path's + ''' + return [self.joinpath(x) for x in self._accessor.listdir(self) if self.joinpath(x).is_dir()] + + def copy(self, destination: Path) -> Path: + '''### copy + ##### Copies the Path to the specificed destination. + + ### Args: + `destination` (Path): Destination to copy to. + + ### Returns: + Path: Path of the new copy. + ''' + shutil.copy(self, destination) + return destination + + @property + def disk_usage(self) -> Path: + return shutil.disk_usage(self) + + def change_suffix(self, newSuffix: str) -> Path: + '''### change_name + ##### Changes the name, including suffix + + ### Args: + `newSuffix` (str): The new suffix + + ### Returns: + Path: Newly named Path. + ''' + return Path(self.parent.joinpath(self.stem + newSuffix)) + + def change_name(self, name: str) -> Path: + '''### change_name + ##### Changes the name, including suffix + + ### Args: + `name` (str): The new name + + ### Returns: + Path: Newly named Path. + ''' + return self.parent.joinpath(name) + + def change_stem(self, new_stem: str) -> Path: + '''### append_stem + ##### Changes the name, ignoring the suffix. + + ### Args: + `append_str` (str): String to append. + + ### Returns: + Path: Newly named Path. + ''' + return self.parent.joinpath(new_stem + self.suffix) + + '''### append_stem + ##### Appends a string to the name, ignoring the suffix. + + ### Args: + `append_str` (str): String to append. + + ### Returns: + Path: Newly named Path. + ''' + + def append_stem(self, append_str: str) -> Path: + '''[summary] + + Arguments: + append_str {str} -- [description] + + Returns: + Path -- [description] + ''' + return self.parent.joinpath(self.stem + append_str + self.suffix) + + def append_name(self, append_str: str): + '''### append_name + ##### Appends a string to the name, including the suffix. + + ### Args: + `append_str` (str): String to append. + + ### Returns: + Path: Newly named Path. + ''' + return Path(self.parent.joinpath(self.name + append_str)) + + def rmtree(self) -> None: + shutil.rmtree(self) + + def size(self) -> int: + return self.stat().st_size + + def createDate(self): + return self.stat().st_ctime + + def modifyDate(self): + return self.stat().st_mtime + + def move(self, destination: Path) -> Path: + '''### move + ##### Moves the Path to a newly specified Location. + + ### Args: + `destination` (Path): The destination to move the file to. + + ### Returns: + Path: The new location of the old Path + ''' + shutil.move(self, destination) + return destination + + @property + def mime(self) -> str: + ext = self.suffix.lstrip(".").lower() + custom_types = { + 'ttf': 'font/ttf', + 'otf': 'font/otf', + } + if ext in custom_types: + return custom_types[ext] + + mime = magic.Magic(mime=True) + mime = mime.from_file(str(self)) + return mime + + def fnmatch(self, match: str) -> bool: + cPath = self.parent + for p in cPath.listall(): + if re.search(re.escape(match).replace("\\*", ".*"), p.name): + return True + + return False + + def joinpath(self, *other): + return Path(super().joinpath(*other)) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self._drv + root = self._root + parts = self._parts + if len(parts) == 1 and (drv or root): + return self + return self._from_parsed_parts(drv, root, parts[:-1]) diff --git a/tools/opus_maker/README.md b/tools/opus_maker/README.md new file mode 100644 index 0000000..e69de29 diff --git a/tools/opus_maker/opus.py b/tools/opus_maker/opus.py new file mode 100644 index 0000000..21faefa --- /dev/null +++ b/tools/opus_maker/opus.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +# Copyright (C) 2019 +# +# This source code is licensed under a +# Creative Commons Attribution-NonCommercial 4.0 International License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# You should have received a copy of the license along with this +# work. If not, see . + + +# opus.py - Simple opus encoder and share-helper +# +# Description: +# +# Simple tool to automatically encode a media file into opus +# with cover-art, metadata and optional defined range (e.g 0:20 to 0:34) +# Most important settings, e.g bitrate/vbr can changed via the config +# The main appeal is the ability to move the output file +# to a specified path/network path (if configured) +# and copy a configured URL with the filename +# to clip-board (if pyperclip is installed) +# +# Syntax/Usage: +# opus [ [ Path: + temp_out_path = file_path.change_name(file_path.stem + "opusthing_xyz.jpg") + cmd = [ + 'ffmpeg', + '-y', + '-i', + str(file_path), + '-an', + '-compression_level', + '75', + '-pix_fmt', + 'yuvj444p', + '-s', + '300x300', + '-c:v', + 'mjpeg', + str(temp_out_path), + ] + _ = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return temp_out_path + + def getFfprobe(self, file_path: Path) -> dict: + cmd = [ + 'ffprobe', + '-show_streams', + '-show_format', + '-print_format', + 'json', + '-loglevel', + 'panic', + str(file_path), + ] + return json.loads(subprocess.check_output(cmd)) + + def getCoverFromFolder(self, file_path: Path) -> Path: + foundFile = None + for file in file_path.parent.listfiles(('.jpg', '.jpeg', '.png', '.webp',)): + if any(w in file.name.lower() for w in ('cover', 'folder', 'artwork')): + foundFile = file + break + elif not foundFile: + foundFile = file + return foundFile + + def hasCover(self, file_path) -> bool: + probe = self.getFfprobe(file_path) + for stream in probe["streams"]: + if stream['codec_type'] == 'video': + if 'tags' in stream and 'comment' in stream['tags']: + if 'cover' in stream['tags']['comment'].lower(): + return True + return False + + def getCoverFromFile(self, file_path: Path) -> Path: + if self.hasCover(file_path): + return self.extractCover(file_path) + return None + + def encodeFile(self, original_file: Path) -> Path: + output_file_path = original_file.parent.joinpath(original_file.name.replace(" ", "_").replace("_-_", "_")).change_suffix(".opus") + if output_file_path.exists(): + _output_file_path = output_file_path + _append_num = 1 + while _output_file_path.exists(): + if _append_num >= 10000: + # Who would ever keep 10000 re-encodes of the same file-name in the same folder!? + raise Exception("Failed to find suitable alternative name for output file") + _output_file_path = _output_file_path.append_stem(F"_{_append_num}") + _append_num += 1 + output_file_path = _output_file_path + + cover: Path = None + if self.withCover: + cover = self.getCoverFromFile(original_file) + if not cover: + cover = self.getCoverFromFolder(original_file) + print(F"Using cover from folder: {cover}") + else: + print(F"Using cover from file: {cover}") + + cmd1 = [ + 'ffmpeg', + '-loglevel', + 'panic', + '-i', + str(original_file), + '-map_metadata', + '0', + ] + if self.startTime: + cmd1.append('-ss') + cmd1.append(self.startTime) + + if self.endTime: + cmd1.append('-to') + cmd1.append(self.endTime) + + cmd1.extend([ + '-f', + 'flac', + '-', + ]) + cmd2 = [ + 'opusenc', + '-', + '--bitrate', + str(self.bitrate), + ] + if self.opusVbr: + cmd2.append('--vbr') + if cover: + cmd2.append('--picture') + cmd2.append(str(cover)) + cmd2.append(str(output_file_path)) + + erange = "" + if start_time and end_time: + erange = F" from {start_time} to {end_time}" + elif start_time: + erange = F" from {start_time} to end" + elif end_time: + erange = F" from start to {end_time}" + + print(F"Encoding{erange} into file {output_file_path}...") + p = subprocess.Popen(cmd1, shell=False, bufsize=0, stderr=subprocess.DEVNULL, stdout=subprocess.PIPE) + p2 = subprocess.Popen(cmd2, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stdin=p.stdout) + stdo, stde = p2.communicate() + + if cover: + cover.unlink() + + return output_file_path + + def mime(self, path: Path) -> str: + ext = path.suffix.lstrip(".").lower() + custom_types = { + 'ttf': 'font/ttf', + 'otf': 'font/otf', + } + if ext in custom_types: + return custom_types[ext] + + mime = magic.Magic(mime=True) + mime = mime.from_file(str(path)) + return mime + + def __init__(self, original_file: Path, + start_time: str = None, + end_time: str = None, + output_dir: Path = None, + copy_link: bool = None, + base_url: str = None, + opus_vbr: bool = None, + bit_rate: int =None, + with_cover: bool = None, + have_mime: bool = None, + ignore_mime: bool = None, + have_pyperclip: bool = None + ) -> None: + errors = [] + # Settings + self.opusVbr = opus_vbr + self.outputDir = output_dir + self.copyLink = copy_link + self.baseUrl = base_url + self.bitrate = bit_rate + self.withCover = with_cover + self.ignoreMime = ignore_mime + + # Commandline args + self.startTime = start_time + self.endTime = end_time + self.originalFile = original_file + + # ### + self.haveMime = have_mime + self.havePyperclip = have_pyperclip + + if not self.bitrate or not isinstance(self.bitrate, (float, int)): + errors.append("The BITRATE setting has to be a number (float or int), default is 64.") + + if not self.opusVbr or not isinstance(self.opusVbr, bool): + errors.append("The OPUS_VBR setting has to be a boolean (True or False), default is True") + + if self.outputDir and not isinstance(self.outputDir, str): + errors.append("The OUTPUT_PATH setting has string of an existing Path or None, default is None") + else: + self.outputDir = Path(output_dir) + if self.outputDir and not self.outputDir.exists(): + errors.append("The OUTPUT_PATH setting has to be an existing proper Path, default is None") + + if not self.withCover and not isinstance(self.withCover, bool): + errors.append("The WITH_COVER setting has to be a boolean (True or False), default is True") + + if self.copyLink and not isinstance(self.copyLink, bool): + errors.append("The COPY_SHARE_LINK setting has to be a boolean (True or False), default is False") + + if self.copyLink and not self.havePyperclip: + errors.append("The COPY_SHARE_LINK requires pyperclip to be installed (pip install pyperclip)") + + if self.copyLink and not self.baseUrl: + errors.append("The COPY_SHARE_LINK setting requires BASE_URL to be set.") + + if len(errors): + print("Incorrect config settings:") + print("\t- " + "\n\t- ".join(errors)) + exit(1) + + try: + self.originalFile = Path(self.originalFile) + except Exception: + print(F"{self.originalFile} is not a valid audio/video file path.") + exit(1) + if not self.originalFile.exists(): + print(F"File {self.originalFile} does not exist.") + exit(1) + + if self.haveMime: + if not self.ignoreMime and not self .mime(self.originalFile).startswith(("audio/", "video/")): + print(F"File {self.originalFile} is no video or audio file.") + exit(1) + + output_file_path = self.encodeFile(self.originalFile) + + if self.outputDir: + print(F"Moving output file to: {self.outputDir}") + new_output_file_path = self.outputDir.joinpath(output_file_path.name) + shutil.move(output_file_path, new_output_file_path) + + if self.copyLink: + url = self.baseUrl.format(file_name=new_output_file_path.name) + print(F"Copying URL to clip-board: {url}") + pyperclip.copy(url) + +class Path(pathlib.Path): # Part of https://gist.github.com/DeadSix27/036810df93804d02b962c0aec8d08b59 + _flavour = pathlib._windows_flavour if os.name == 'nt' else pathlib._posix_flavour + + def __new__(cls, *args): + return super(Path, cls).__new__(cls, *args) + + def __init__(self, *args): + super().__init__() + self.ssuffix = self.suffix.lstrip(".") + self._some_instance_ppath_value = self.exists() + + def listfiles(self, extensions=()) -> List[Path]: + '''### listfiles + ##### listfiles + + ### Args: + `extensions` (tuple, optional): List of extensions to limit listing to, with dot prefix. Defaults to (). + + ### Returns: + List[Path]: List of Paths, matching the optionally specificed extension(s) + ''' + lst = None + if len(extensions) > 0: + lst = [self.joinpath(x) for x in self._accessor.listdir(self) if self.joinpath(x).is_file() and x.lower().endswith(extensions)] + else: + lst = [self.joinpath(x) for x in self._accessor.listdir(self) if self.joinpath(x).is_file()] + + def convert(text): + return int(text) if text.isdigit() else text + + def alphanum_key(key): + return [convert(c) for c in re.split('([0-9]+)', str(key))] + + lst = sorted(lst, key=alphanum_key) + return lst + + def change_suffix(self, newSuffix: str) -> Path: + '''### change_name + ##### Changes the name, including suffix + + ### Args: + `newSuffix` (str): The new suffix + + ### Returns: + Path: Newly named Path. + ''' + return Path(self.parent.joinpath(self.stem + newSuffix)) + + def change_name(self, name: str) -> Path: + '''### change_name + ##### Changes the name, including suffix + + ### Args: + `name` (str): The new name + + ### Returns: + Path: Newly named Path. + ''' + return self.parent.joinpath(name) + + def append_stem(self, append_str: str) -> Path: + '''### append_stem + ##### Appends a string to the stem, excluding the suffix. + + ### Args: + `append_str` (str): String to append. + + ### Returns: + Path: Newly named Path. + ''' + return self.parent.joinpath(self.stem + append_str + self.suffix) + +if __name__ == "__main__": + try: + import magic + HAVE_MIME = True + except ImportError: + HAVE_MIME = False + + try: + import pyperclip + HAVE_PYPERCLIP = True + except ImportError: + HAVE_PYPERCLIP = False + + if len(sys.argv) >= 2: + file_path = sys.argv[1] + start_time = None + end_time = None + if len(sys.argv) == 3: + start_time = sys.argv[2] + elif len(sys.argv) >= 4: + start_time = sys.argv[2] + end_time = sys.argv[3] + + OpusMaker(file_path, start_time, end_time, + output_dir=OUTPUT_PATH, + copy_link=COPY_SHARE_LINK, + base_url=BASE_URL, + opus_vbr=OPUS_VBR, + bit_rate=BITRATE, + with_cover=WITH_COVER, + have_mime=HAVE_MIME, + ignore_mime=IGNORE_MIME, + have_pyperclip=HAVE_PYPERCLIP, + ) + else: + print("Syntax: opus [ [