Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel download #289

Closed
wants to merge 10 commits into from
Closed

Parallel download #289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4bc8cbf
Basic barebone commit. Requires debugging
Aayush2492 Aug 4, 2022
5b9b536
Clean up
Aayush2492 Aug 4, 2022
05c3769
Requirement fix
Aayush2492 Aug 4, 2022
ae84e88
Fix requirements.txt
Aayush2492 Aug 4, 2022
8fed1f3
Reverting requirements as multiprocesssing library presentby default
Aayush2492 Aug 4, 2022
bb9f342
Comment out rich.progess since its not compatible with multiprocessi…
Aayush2492 Aug 5, 2022
78ef4d9
Remove unnecessary imports
Aayush2492 Aug 5, 2022
36d05aa
Remove commented blocks
Aayush2492 Aug 8, 2022
9be56c6
Make formatting compliant with the DeepSource specification
Aayush2492 Aug 8, 2022
bcf3ab1
Fix linting errors
Aayush2492 Aug 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ sentry_sdk~=1.5
yt-dlp>=2022.01.21
spotipy~=2.19
mutagen~=1.45
rich~=12.0
rich~=12.0
143 changes: 68 additions & 75 deletions spotify_dl/spotify.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

from spotify_dl.scaffold import log
from spotify_dl.utils import sanitize
from rich.progress import Progress


def fetch_tracks(sp, item_type, url):
"""
Fetches tracks from the provided URL.
Expand All @@ -15,85 +16,77 @@ def fetch_tracks(sp, item_type, url):
offset = 0

if item_type == 'playlist':
with Progress() as progress:
songs_task = progress.add_task(description="Fetching songs from playlist..")
while True:
items = sp.playlist_items(playlist_id=url,
fields='items.track.name,items.track.artists(name, uri),'
'items.track.album(name, release_date, total_tracks, images),'
'items.track.track_number,total, next,offset,'
'items.track.id',
additional_types=['track'], offset=offset)
total_songs = items.get('total')
track_info_task = progress.add_task(description="Fetching track info", total=len(items['items']))
for item in items['items']:
track_info = item.get('track')
# If the user has a podcast in their playlist, there will be no track
# Without this conditional, the program will fail later on when the metadata is fetched
if track_info is None:
offset += 1
continue
track_album_info = track_info.get('album')
track_num = track_info.get('track_number')
spotify_id = track_info.get('id')
track_name = track_info.get('name')
track_artist = ", ".join([artist['name'] for artist in track_info.get('artists')])
if track_album_info:
track_album = track_album_info.get('name')
track_year = track_album_info.get('release_date')[:4] if track_album_info.get('release_date') else ''
album_total = track_album_info.get('total_tracks')
if len(item['track']['album']['images']) > 0:
cover = item['track']['album']['images'][0]['url']
else:
cover = None

artists = track_info.get('artists')
main_artist_id = artists[0].get('uri', None) if len(artists) > 0 else None
genres = sp.artist(artist_id=main_artist_id).get('genres', []) if main_artist_id else []
if len(genres) > 0:
genre = genres[0]
else:
genre = ""
songs_list.append({"name": track_name, "artist": track_artist, "album": track_album, "year": track_year,
"num_tracks": album_total, "num": track_num, "playlist_num": offset + 1,
"cover": cover, "genre": genre, "spotify_id": spotify_id})
while True:
items = sp.playlist_items(playlist_id=url,
fields='items.track.name,items.track.artists(name, uri),'
'items.track.album(name, release_date, total_tracks, images),'
'items.track.track_number,total, next,offset,'
'items.track.id',
additional_types=['track'], offset=offset)
total_songs = items.get('total')
for item in items['items']:
track_info = item.get('track')
# If the user has a podcast in their playlist, there will be no track
# Without this conditional, the program will fail later on when the metadata is fetched
if track_info is None:
offset += 1
progress.update(task_id=track_info_task, description=f"Fetching track info for \n{track_name}",advance=1)
progress.update(task_id=songs_task, description=f"Fetched {offset} of {total_songs} songs from the playlist", advance=100, total=total_songs)
if total_songs == offset:
break

elif item_type == 'album':
with Progress() as progress:
album_songs_task = progress.add_task(description="Fetching songs from the album..")
while True:
album_info = sp.album(album_id=url)
items = sp.album_tracks(album_id=url, offset=offset)
total_songs = items.get('total')
track_album = album_info.get('name')
track_year = album_info.get('release_date')[:4] if album_info.get('release_date') else ''
album_total = album_info.get('total_tracks')
if len(album_info['images']) > 0:
cover = album_info['images'][0]['url']
continue
track_album_info = track_info.get('album')
track_num = track_info.get('track_number')
spotify_id = track_info.get('id')
track_name = track_info.get('name')
track_artist = ", ".join([artist['name'] for artist in track_info.get('artists')])
if track_album_info:
track_album = track_album_info.get('name')
track_year = track_album_info.get('release_date')[:4] if track_album_info.get('release_date') else ''
album_total = track_album_info.get('total_tracks')
if len(item['track']['album']['images']) > 0:
cover = item['track']['album']['images'][0]['url']
else:
cover = None
if len(sp.artist(artist_id=album_info['artists'][0]['uri'])['genres']) > 0:
genre = sp.artist(artist_id=album_info['artists'][0]['uri'])['genres'][0]

artists = track_info.get('artists')
main_artist_id = artists[0].get('uri', None) if len(artists) > 0 else None
genres = sp.artist(artist_id=main_artist_id).get('genres', []) if main_artist_id else []
if len(genres) > 0:
genre = genres[0]
else:
genre = ""
for item in items['items']:
track_name = item.get('name')
track_artist = ", ".join([artist['name'] for artist in item['artists']])
track_num = item['track_number']
spotify_id = item.get('id')
songs_list.append({"name": track_name, "artist": track_artist, "album": track_album, "year": track_year,
"num_tracks": album_total, "num": track_num, "playlist_num": offset + 1,
"cover": cover, "genre": genre, "spotify_id": spotify_id})
offset += 1
genre = ""
songs_list.append({"name": track_name, "artist": track_artist, "album": track_album, "year": track_year,
"num_tracks": album_total, "num": track_num, "playlist_num": offset + 1,
"cover": cover, "genre": genre, "spotify_id": spotify_id})
offset += 1
if total_songs == offset:
break

elif item_type == 'album':
while True:
album_info = sp.album(album_id=url)
items = sp.album_tracks(album_id=url, offset=offset)
total_songs = items.get('total')
track_album = album_info.get('name')
track_year = album_info.get('release_date')[:4] if album_info.get('release_date') else ''
album_total = album_info.get('total_tracks')
if len(album_info['images']) > 0:
cover = album_info['images'][0]['url']
else:
cover = None
if len(sp.artist(artist_id=album_info['artists'][0]['uri'])['genres']) > 0:
genre = sp.artist(artist_id=album_info['artists'][0]['uri'])['genres'][0]
else:
genre = ""
for item in items['items']:
track_name = item.get('name')
track_artist = ", ".join([artist['name'] for artist in item['artists']])
track_num = item['track_number']
spotify_id = item.get('id')
songs_list.append({"name": track_name, "artist": track_artist, "album": track_album, "year": track_year,
"num_tracks": album_total, "num": track_num, "playlist_num": offset + 1,
"cover": cover, "genre": genre, "spotify_id": spotify_id})
offset += 1

progress.update(task_id=album_songs_task, description=f"Fetched {offset} of {album_total} songs from the album {track_album}", advance=offset, total=album_total)
if album_total == offset:
break
if album_total == offset:
break

elif item_type == 'track':
items = sp.track(track_id=url)
Expand Down
60 changes: 36 additions & 24 deletions spotify_dl/spotify_dl.py
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
import json
from multiprocessing.dummy import Process
import os
import sys
from logging import DEBUG
Expand Down Expand Up @@ -74,32 +75,43 @@ def spotify_dl():

C_ID, C_SECRET = tokens
sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials(client_id=C_ID,client_secret=C_SECRET))
log.debug('Arguments: {}'.format(args))
log.debug(f'Arguments: {args}')

processes = []
for url in args.url:
if url:
valid_item = validate_spotify_url(url)

if not valid_item:
sys.exit(1)



if args.output:
item_type, item_id = parse_spotify_url(url)
directory_name = get_item_name(sp, item_type, item_id)
save_path = Path(PurePath.joinpath(Path(args.output), Path(directory_name)))
save_path.mkdir(parents=True, exist_ok=True)
console.print(f"Saving songs to [bold green]{directory_name}[/bold green] directory")
songs = fetch_tracks(sp, item_type, url)
else:
songs = {}
if args.download is True:
file_name_f = default_filename
if args.keep_playlist_order:
file_name_f = playlist_num_filename
if save_path is not None:
download_songs(songs, save_path, args.format_str, args.skip_mp3, args.keep_playlist_order, args.no_overwrites, args.skip_non_music_sections, file_name_f)
processes.append(Process(target=download_for_one_link, args=(url, args, sp)))
processes[-1].start()

for process in processes:
process.join()


def download_for_one_link(url, args, sp) :
"""
Downloads songs for one playlist/album/track url.
This function with the required arg can be passed to
Process constructor to initiate parallel download.
"""
if url:
valid_item = validate_spotify_url(url)

if not valid_item:
sys.exit(1)

if args.output:
item_type, item_id = parse_spotify_url(url)
directory_name = get_item_name(sp, item_type, item_id)
save_path = Path(PurePath.joinpath(Path(args.output), Path(directory_name)))
save_path.mkdir(parents=True, exist_ok=True)
console.print(f"Saving songs to [bold green]{directory_name}[/bold green] directory")

songs = fetch_tracks(sp, item_type, url)
if args.download is True:
file_name_f = default_filename
if args.keep_playlist_order:
file_name_f = playlist_num_filename

download_songs(songs, save_path, args.format_str, args.skip_mp3, args.keep_playlist_order, args.no_overwrites, args.skip_non_music_sections, file_name_f)


if __name__ == '__main__':
Expand Down