From fe02d944ada41363efc9ec58ffb0d4f8d729fdf2 Mon Sep 17 00:00:00 2001 From: Gustav Andreasson Date: Thu, 9 Nov 2023 13:29:05 +0100 Subject: [PATCH] Sanitize input to rate endpoint. Lint all python files --- frontend/urls.py | 3 +- frontend/views.py | 2 +- records/admin.py | 58 +++--- records/apps.py | 2 +- records/management/commands/downloadcovers.py | 21 +- records/management/commands/readartistxml.py | 65 +++--- records/management/commands/updaterecords.py | 6 +- records/models.py | 69 +++---- records/progress.py | 2 +- records/services/artist.py | 99 +++++---- records/services/collection.py | 71 ++++--- records/services/rates.py | 8 +- records/services/record.py | 192 ++++++++---------- records/spotify.py | 31 +-- records/templates/records/index.html | 165 ++++++++------- records/urls.py | 42 ++-- records/views/artist.py | 49 ++--- records/views/collection.py | 40 ++-- records/views/rates.py | 8 +- records/views/record.py | 3 +- records2/settings.py | 179 ++++++++-------- records2/urls.py | 6 +- 22 files changed, 529 insertions(+), 592 deletions(-) diff --git a/frontend/urls.py b/frontend/urls.py index 849d9a7..268cfbc 100644 --- a/frontend/urls.py +++ b/frontend/urls.py @@ -1,5 +1,6 @@ from django.conf.urls import url from . import views + urlpatterns = [ - url(r'^$', views.index), + url(r"^$", views.index), ] diff --git a/frontend/views.py b/frontend/views.py index 9bb811e..c94878a 100644 --- a/frontend/views.py +++ b/frontend/views.py @@ -5,4 +5,4 @@ def index(request): if request.session.is_empty(): request.session.create() # Create session cookie if it does not exist - return render(request, 'frontend/index.html') + return render(request, "frontend/index.html") diff --git a/records/admin.py b/records/admin.py index c250073..b36f06d 100644 --- a/records/admin.py +++ b/records/admin.py @@ -1,23 +1,15 @@ from django.contrib import admin -from records.models import ( - Record, - Artist, - RecordArtists, - Track, - Listen, - RecordListens, - ArtistMembers, - DiscogsUser -) +from records.models import Record, Artist, RecordArtists, Track, Listen, RecordListens, ArtistMembers, DiscogsUser from records.services.record import updateRecord from records.services.artist import updateArtist from django.core.cache import cache + # Register your models here. class ArtistInline(admin.TabularInline): model = RecordArtists - readonly_fields = ('artist',) + readonly_fields = ("artist",) extra = 0 @@ -45,12 +37,15 @@ def clear_cache_item(modeladmin, request, queryset): class RecordAdmin(admin.ModelAdmin): - fields = ['id', 'name', 'master', 'year', 'format', - 'cover', 'thumbnail', 'price', 'updated'] + fields = ["id", "name", "master", "year", "format", "cover", "thumbnail", "price", "updated"] inlines = [ArtistInline, TrackInline, ListenInline] - list_display = ('id', 'get_artist', 'name', 'format', 'updated') - search_fields = ['name'] - actions = [reset_updated, update_record, clear_cache_item, ] + list_display = ("id", "get_artist", "name", "format", "updated") + search_fields = ["name"] + actions = [ + reset_updated, + update_record, + clear_cache_item, + ] admin.site.register(Record, RecordAdmin) @@ -63,43 +58,44 @@ def update_artist(modeladmin, request, queryset): class MembersInline(admin.TabularInline): model = ArtistMembers - readonly_fields = ('member',) + readonly_fields = ("member",) extra = 0 - fk_name = 'group' + fk_name = "group" class GroupsInline(admin.TabularInline): model = ArtistMembers - readonly_fields = ('group',) + readonly_fields = ("group",) extra = 0 - fk_name = 'member' + fk_name = "member" class ArtistAdmin(admin.ModelAdmin): - fields = ['id', 'name', 'description', - 'image', 'updated', 'collectionUpdated'] + fields = ["id", "name", "description", "image", "updated", "collectionUpdated"] inlines = [MembersInline, GroupsInline] - list_display = ('id', 'name', 'description', 'image', - 'updated', 'collectionUpdated') - search_fields = ['name'] - actions = [reset_updated, update_artist, ] + list_display = ("id", "name", "description", "image", "updated", "collectionUpdated") + search_fields = ["name"] + actions = [ + reset_updated, + update_artist, + ] admin.site.register(Artist, ArtistAdmin) class ListenAdmin(admin.ModelAdmin): - list_display = ('name', 'icon', 'template') - search_fields = ['name'] + list_display = ("name", "icon", "template") + search_fields = ["name"] admin.site.register(Listen, ListenAdmin) class DiscogsUserAdmin(admin.ModelAdmin): - fields = ['username'] - list_display = ('id', 'username') - search_fields = ['username'] + fields = ["username"] + list_display = ("id", "username") + search_fields = ["username"] admin.site.register(DiscogsUser, DiscogsUserAdmin) diff --git a/records/apps.py b/records/apps.py index 857039c..ff12aae 100644 --- a/records/apps.py +++ b/records/apps.py @@ -2,7 +2,7 @@ class RecordsConfig(AppConfig): - name = 'records' + name = "records" def ready(self): from . import signals diff --git a/records/management/commands/downloadcovers.py b/records/management/commands/downloadcovers.py index c52b3d0..d618e1a 100644 --- a/records/management/commands/downloadcovers.py +++ b/records/management/commands/downloadcovers.py @@ -8,12 +8,10 @@ class Command(BaseCommand): help = "Downloads covers to all records" def add_arguments(self, parser): - parser.add_argument('limit', type=int, - help="Limit number of downloaded covers") + parser.add_argument("limit", type=int, help="Limit number of downloaded covers") def handle(self, *args, **options): - records = Record.objects.filter(cover_file__exact='')[ - :options['limit']] + records = Record.objects.filter(cover_file__exact="")[: options["limit"]] for record in records: self.stdout.write("Downloading cover for record " + record.name) try: @@ -21,20 +19,17 @@ def handle(self, *args, **options): record.save() except HTTPError as e: if e.response.status_code == 429: - self.stdout.write("Discogs download limit reached. Wait" - + " for a minute before running again") + self.stdout.write("Discogs download limit reached. Wait" + " for a minute before running again") break elif e.response.status_code == 404: - self.stdout.write("Cover file for record " + record.name - + " is missing on discogs. Updating " - + "record.\n") + self.stdout.write( + "Cover file for record " + record.name + " is missing on discogs. Updating " + "record.\n" + ) updateRecord(record) continue else: - self.stdout.write("Error downloading cover for record " - + record.name + "\n" + str(e)) + self.stdout.write("Error downloading cover for record " + record.name + "\n" + str(e)) continue except RequestException as e: - self.stdout.write("Error downloading cover for record " - + record.name + "\n" + str(e)) + self.stdout.write("Error downloading cover for record " + record.name + "\n" + str(e)) break diff --git a/records/management/commands/readartistxml.py b/records/management/commands/readartistxml.py index 9b614e1..ee942b4 100644 --- a/records/management/commands/readartistxml.py +++ b/records/management/commands/readartistxml.py @@ -8,26 +8,23 @@ class Command(BaseCommand): help = "Reads artist datadump xml and creates all artists" def add_arguments(self, parser): - parser.add_argument('file', type=str, - help="XML file to import") - parser.add_argument('-s', '--start', type=int, - help="Position of first artist to import") - parser.add_argument('-e', '--end', type=int, - help="Position of last artist to import") + parser.add_argument("file", type=str, help="XML file to import") + parser.add_argument("-s", "--start", type=int, help="Position of first artist to import") + parser.add_argument("-e", "--end", type=int, help="Position of last artist to import") def handle(self, *args, **options): self.stdout.write("Counting number of artists in file...") - with open(options['file'], 'r') as f: - nr_artists = sum(line.count('') for line in f) - start = options['start'] if options['start'] else 0 - end = min(options['end'], nr_artists) if options['end'] else nr_artists + with open(options["file"], "r") as f: + nr_artists = sum(line.count("") for line in f) + start = options["start"] if options["start"] else 0 + end = min(options["end"], nr_artists) if options["end"] else nr_artists if start > end: self.stdout.write("No artists in range") return - if options['start'] or options['end']: - self.stdout.write("Importing artists " + str(start) + " to " - + str(end) + " of " + str(nr_artists) - + " artists") + if options["start"] or options["end"]: + self.stdout.write( + "Importing artists " + str(start) + " to " + str(end) + " of " + str(nr_artists) + " artists" + ) nr_artists = end - start + 1 else: self.stdout.write("Importing " + str(nr_artists) + " artists") @@ -35,34 +32,34 @@ def handle(self, *args, **options): artists_imported = 0 loop_counter = 0 batch_size = 1000 - context = etree.iterparse( - options['file'], events=('end',), tag='artist') + context = etree.iterparse(options["file"], events=("end",), tag="artist") artists_batch = [] for event, elem in context: if loop_counter >= start: - if elem.find('name').text and elem.find('id').text and \ - not elem.find('data_quality').text in \ - ('Needs Major Changes', 'Entirely Incorrect'): - name = fixArtistName(elem.find('name').text) + if ( + elem.find("name").text + and elem.find("id").text + and not elem.find("data_quality").text in ("Needs Major Changes", "Entirely Incorrect") + ): + name = fixArtistName(elem.find("name").text) sname = name[:20].lower() - artists_batch.append(Artist(id=elem.find('id').text, - name=name, - sname=sname)) + artists_batch.append(Artist(id=elem.find("id").text, name=name, sname=sname)) if len(artists_batch) >= batch_size: - Artist.objects.bulk_create(artists_batch, - ignore_conflicts=True) + Artist.objects.bulk_create(artists_batch, ignore_conflicts=True) artists_batch = [] artists_imported += batch_size artist_counter += 1 if artist_counter % 863 == 1 or artist_counter == nr_artists: progress = (artist_counter / nr_artists) * 100 self.stdout.write( - "[{0}] {1}% {2}/{3}".format('#'*int(progress/2) - + ' '*(50-int(progress/2)), - int(progress), - artist_counter, - nr_artists), - ending='\r') + "[{0}] {1}% {2}/{3}".format( + "#" * int(progress / 2) + " " * (50 - int(progress / 2)), + int(progress), + artist_counter, + nr_artists, + ), + ending="\r", + ) self.stdout.flush() elem.clear() while elem.getprevious() is not None: @@ -71,8 +68,6 @@ def handle(self, *args, **options): if loop_counter > end: break if artists_batch: - Artist.objects.bulk_create(artists_batch, - ignore_conflicts=True) + Artist.objects.bulk_create(artists_batch, ignore_conflicts=True) artists_imported += len(artists_batch) - self.stdout.write("\nArtist import done. " + str(artists_imported) - + " artists imported.") + self.stdout.write("\nArtist import done. " + str(artists_imported) + " artists imported.") diff --git a/records/management/commands/updaterecords.py b/records/management/commands/updaterecords.py index fce39ec..33d4a65 100644 --- a/records/management/commands/updaterecords.py +++ b/records/management/commands/updaterecords.py @@ -7,12 +7,10 @@ class Command(BaseCommand): help = "Loads extra info to all records" def add_arguments(self, parser): - parser.add_argument('limit', type=int, - help="Limit number of updated records") + parser.add_argument("limit", type=int, help="Limit number of updated records") def handle(self, *args, **options): - records = Record.objects.filter(updated__isnull=True)[ - :options['limit']] + records = Record.objects.filter(updated__isnull=True)[: options["limit"]] for record in records: self.stdout.write("Updating record " + record.name) updateRecord(record) diff --git a/records/models.py b/records/models.py index 1b6e200..37bfefe 100644 --- a/records/models.py +++ b/records/models.py @@ -20,8 +20,7 @@ class Artist(models.Model): image = models.CharField(max_length=255, blank=True, null=True) updated = models.DateField(blank=True, null=True) collectionUpdated = models.DateField(blank=True, null=True) - members = models.ManyToManyField( - 'self', through='ArtistMembers', symmetrical=False) + members = models.ManyToManyField("self", through="ArtistMembers", symmetrical=False) def __str__(self): return self.name @@ -30,14 +29,11 @@ def to_dict(self, full=True): if not full: return {"id": self.id, "name": self.name} member_relations = ArtistMembers.objects.filter(group=self) - members = [{"artist": mr.member.to_dict( - False), "active": mr.active} for mr in member_relations] + members = [{"artist": mr.member.to_dict(False), "active": mr.active} for mr in member_relations] group_relations = ArtistMembers.objects.filter(member=self) - groups = [{"artist": gr.group.to_dict( - False), "active": gr.active} for gr in group_relations] + groups = [{"artist": gr.group.to_dict(False), "active": gr.active} for gr in group_relations] updated = str(self.updated) if self.updated else None - collectionUpdated = str( - self.collectionUpdated) if self.collectionUpdated else None + collectionUpdated = str(self.collectionUpdated) if self.collectionUpdated else None return { "id": self.id, "name": self.name, @@ -46,7 +42,7 @@ def to_dict(self, full=True): "members": members, "groups": groups, "updated": updated, - "collectionUpdated": collectionUpdated + "collectionUpdated": collectionUpdated, } def save(self, *args, **kwargs): @@ -54,7 +50,7 @@ def save(self, *args, **kwargs): super(Artist, self).save(*args, **kwargs) class Meta: - indexes = [models.Index(fields=['sname'])] + indexes = [models.Index(fields=["sname"])] class Listen(models.Model): @@ -75,15 +71,13 @@ class Record(models.Model): year = models.IntegerField(blank=True, null=True) updated = models.DateField(blank=True, null=True) thumbnail = models.CharField(max_length=255, blank=True, null=True) - price = models.DecimalField( - max_digits=7, decimal_places=2, blank=True, null=True) - listens = models.ManyToManyField(Listen, through='RecordListens') - artists = models.ManyToManyField(Artist, through='RecordArtists') - cover_file = models.ImageField( - upload_to="records", blank=True, null=True) + price = models.DecimalField(max_digits=7, decimal_places=2, blank=True, null=True) + listens = models.ManyToManyField(Listen, through="RecordListens") + artists = models.ManyToManyField(Artist, through="RecordArtists") + cover_file = models.ImageField(upload_to="records", blank=True, null=True) thumbnail_file = ImageSpecField( - source='cover_file', processors=[ResizeToFit(200, 90)], - format='JPEG', options={'quality': 100}) + source="cover_file", processors=[ResizeToFit(200, 90)], format="JPEG", options={"quality": 100} + ) def __str__(self): return self.name @@ -91,7 +85,7 @@ def __str__(self): def to_dict(self): cached_data = cache.get(self.get_cache_key()) if cached_data: - cached_data['cached'] = True + cached_data["cached"] = True return cached_data dict = { "id": self.id, @@ -99,14 +93,13 @@ def to_dict(self): "master": self.master, "cover": self.cover_file.url if self.cover_file else self.cover, "format": self.format, - "year": self.year, + "year": self.year, "thumbnail": self.thumbnail_file.url if self.thumbnail_file else self.thumbnail, "price": str(self.price) if self.price else None, - "updated": str(self.updated) if self.updated else None + "updated": str(self.updated) if self.updated else None, } ras = RecordArtists.objects.filter(record=self) - artists = [{"artist": ra.artist.to_dict( - False), "delimiter": ra.delimiter} for ra in ras] + artists = [{"artist": ra.artist.to_dict(False), "delimiter": ra.delimiter} for ra in ras] dict["artists"] = artists rls = RecordListens.objects.filter(record=self) listens = [rl.to_dict() for rl in rls] @@ -118,7 +111,7 @@ def to_dict(self): return dict def get_cache_key(self): - return 'record-' + str(self.id) + return "record-" + str(self.id) def get_artist(self): ras = RecordArtists.objects.filter(record=self) @@ -126,12 +119,13 @@ def get_artist(self): for ra in ras: artists += ra.artist.name + " " return artists + get_artist.short_description = "Artist" class DiscogsUser(models.Model): username = models.CharField(max_length=255) - records = models.ManyToManyField(Record, through='UserRecords') + records = models.ManyToManyField(Record, through="UserRecords") def __str__(self): return self.username @@ -143,29 +137,24 @@ class UserRecords(models.Model): added_date = models.DateField(blank=True, null=True) class Meta: - ordering = ['id'] + ordering = ["id"] class Track(models.Model): position = models.CharField(max_length=255, blank=True, null=True) name = models.CharField(max_length=1024, blank=True, null=True) record = models.ForeignKey(Record, on_delete=models.CASCADE) - artists = models.ManyToManyField(Artist, through='TrackArtists') + artists = models.ManyToManyField(Artist, through="TrackArtists") def __str__(self): return self.name def to_dict(self): tas = TrackArtists.objects.filter(track=self) - artists = [{"artist": ta.artist.to_dict( - False), "delimiter": ta.delimiter} for ta in tas] + artists = [{"artist": ta.artist.to_dict(False), "delimiter": ta.delimiter} for ta in tas] if len(artists) == 0: artists = None - return { - "position": self.position, - "name": self.name, - "artists": artists - } + return {"position": self.position, "name": self.name, "artists": artists} class RecordListens(models.Model): @@ -179,7 +168,7 @@ def to_dict(self): "type": self.listen.name, "name": self.name, "icon": self.listen.icon, - "html": self.listen.template.format(self.listen_key) + "html": self.listen.template.format(self.listen_key), } @@ -190,14 +179,12 @@ class RecordArtists(models.Model): position = models.IntegerField(blank=True, null=True) class Meta: - ordering = ['position'] + ordering = ["position"] class ArtistMembers(models.Model): - group = models.ForeignKey( - Artist, on_delete=models.CASCADE, related_name='group') - member = models.ForeignKey( - Artist, on_delete=models.CASCADE, related_name='member') + group = models.ForeignKey(Artist, on_delete=models.CASCADE, related_name="group") + member = models.ForeignKey(Artist, on_delete=models.CASCADE, related_name="member") active = models.IntegerField(blank=True, null=True) @@ -208,7 +195,7 @@ class TrackArtists(models.Model): position = models.IntegerField(blank=True, null=True) class Meta: - ordering = ['position'] + ordering = ["position"] class DiscogsAccess(models.Model): diff --git a/records/progress.py b/records/progress.py index da02c79..f00493f 100644 --- a/records/progress.py +++ b/records/progress.py @@ -41,4 +41,4 @@ def getProgress(): def __cacheKey(): - return 'progress-' + session + return "progress-" + session diff --git a/records/services/artist.py b/records/services/artist.py index 545f518..68def6d 100644 --- a/records/services/artist.py +++ b/records/services/artist.py @@ -11,12 +11,9 @@ def createArtist(id, name): - artist, created = Artist.objects.get_or_create( - id=id, - defaults={'name': fixArtistName(name)}) + artist, created = Artist.objects.get_or_create(id=id, defaults={"name": fixArtistName(name)}) if created: - logger.info("Created artist " + artist.name - + " (" + str(artist.id) + ")") + logger.info("Created artist " + artist.name + " (" + str(artist.id) + ")") return artist @@ -24,80 +21,76 @@ def updateArtist(artist): logger.info("Updating artist " + artist.name + " (" + str(artist.id) + ")") try: artist_data = discogs.getArtist(artist.id) - artist.description = artist_data.get('profile') - if artist_data.get('images'): - artist.image = artist_data['images'][0].get('resource_url') - if artist_data.get('members'): - for member_data in artist_data.get('members'): + artist.description = artist_data.get("profile") + if artist_data.get("images"): + artist.image = artist_data["images"][0].get("resource_url") + if artist_data.get("members"): + for member_data in artist_data.get("members"): member, created = Artist.objects.get_or_create( - id=member_data['id'], - defaults={'name': fixArtistName(member_data['name'])}) + id=member_data["id"], defaults={"name": fixArtistName(member_data["name"])} + ) if created: - logger.info("Created artist " + member.name - + " (" + str(member.id) + ")") + logger.info("Created artist " + member.name + " (" + str(member.id) + ")") ArtistMembers.objects.update_or_create( - group=artist, - member=member, - defaults={'active': member_data['active']}) - if artist_data.get('groups'): - for group_data in artist_data.get('groups'): + group=artist, member=member, defaults={"active": member_data["active"]} + ) + if artist_data.get("groups"): + for group_data in artist_data.get("groups"): group, created = Artist.objects.get_or_create( - id=group_data['id'], - defaults={'name': fixArtistName(group_data['name'])}) + id=group_data["id"], defaults={"name": fixArtistName(group_data["name"])} + ) if created: - logger.info("Created artist " + group.name - + " (" + str(group.id) + ")") + logger.info("Created artist " + group.name + " (" + str(group.id) + ")") ArtistMembers.objects.update_or_create( - group=group, - member=artist, - defaults={'active': group_data['active']}) + group=group, member=artist, defaults={"active": group_data["active"]} + ) artist.updated = date.today() artist.save() except discogs.DiscogsError as de: - logger.info("Did not find artist " + artist.name - + " (" + str(artist.id) + ") on discogs\n" + str(de)) + logger.info("Did not find artist " + artist.name + " (" + str(artist.id) + ") on discogs\n" + str(de)) return False except DatabaseError as de: - logger.error("Could not update artist " + artist.name - + " (" + str(artist.id) + ")\n" + str(de)) + logger.error("Could not update artist " + artist.name + " (" + str(artist.id) + ")\n" + str(de)) return False return True def collectArtistReleases(artist): - logger.info("Collecting releases for artist " - + artist.name + " (" + str(artist.id) + ")") + logger.info("Collecting releases for artist " + artist.name + " (" + str(artist.id) + ")") try: artist_releases = discogs.getArtistReleases(artist.id) - artist_main_releases = [ - release for release in artist_releases - if release.get('role') == "Main"] + artist_main_releases = [release for release in artist_releases if release.get("role") == "Main"] tot = len(artist_main_releases) nr = 0 for release_data in artist_main_releases: try: - recordService.createRecord(release_data['id'], { - 'name': release_data.get('title'), - 'cover': release_data.get('thumb'), - 'thumbnail': release_data.get('thumb'), - 'year': release_data.get('year'), - 'type': release_data.get('type'), - 'main_release': release_data.get('main_release'), - 'artists': [artist] - }) + recordService.createRecord( + release_data["id"], + { + "name": release_data.get("title"), + "cover": release_data.get("thumb"), + "thumbnail": release_data.get("thumb"), + "year": release_data.get("year"), + "type": release_data.get("type"), + "main_release": release_data.get("main_release"), + "artists": [artist], + }, + ) except DatabaseError as de: logger.error( "Could not create record " - + release_data.get('title') - + " (" + release_data['id'] + ")\n" - + str(de)) + + release_data.get("title") + + " (" + + release_data["id"] + + ")\n" + + str(de) + ) nr = nr + 1 if nr % 10 == 0: - progress.updateProgress('create', int((nr * 100) / tot)) - progress.updateProgress('create', 100) + progress.updateProgress("create", int((nr * 100) / tot)) + progress.updateProgress("create", 100) except discogs.DiscogsError as de: - logger.info("Did not find releases for " - + artist.name + " on discogs\n" + str(de)) + logger.info("Did not find releases for " + artist.name + " on discogs\n" + str(de)) return False artist.collectionUpdated = date.today() artist.save() @@ -105,5 +98,5 @@ def collectArtistReleases(artist): def fixArtistName(name): - myre = re.compile('\\(\\d+\\)$') - return myre.sub('', name).strip() + myre = re.compile("\\(\\d+\\)$") + return myre.sub("", name).strip() diff --git a/records/services/collection.py b/records/services/collection.py index e5a8061..100d549 100644 --- a/records/services/collection.py +++ b/records/services/collection.py @@ -12,54 +12,59 @@ def updateCollection(user): logger.info("Updating collection for " + user.username) try: collection_data = discogs.getCollection(user.username) - old_collection = list(UserRecords.objects.filter(user=user) - .values_list('record_id', flat=True)) + old_collection = list(UserRecords.objects.filter(user=user).values_list("record_id", flat=True)) tot = len(collection_data) nr = 0 for release_data in collection_data: - basic_information = release_data['basic_information'] - if basic_information['id'] not in old_collection: + basic_information = release_data["basic_information"] + if basic_information["id"] not in old_collection: try: record = recordService.createRecord( - basic_information['id'], { - 'name': basic_information.get('title'), - 'master': basic_information.get('master_id'), - 'cover': basic_information.get('cover_image'), - 'thumbnail': basic_information.get('thumb'), - 'year': basic_information.get('year'), - 'format': basic_information.get('formats'), - 'artists': basic_information['artists'] - }) - ur = UserRecords.objects\ - .create(user=user, - record=record, - added_date=release_data['date_added'][0:10]) + basic_information["id"], + { + "name": basic_information.get("title"), + "master": basic_information.get("master_id"), + "cover": basic_information.get("cover_image"), + "thumbnail": basic_information.get("thumb"), + "year": basic_information.get("year"), + "format": basic_information.get("formats"), + "artists": basic_information["artists"], + }, + ) + ur = UserRecords.objects.create( + user=user, record=record, added_date=release_data["date_added"][0:10] + ) logger.info( - "Added record " + record.name - + " (" + str(record.id) + ") to collection for " - + user.username) + "Added record " + record.name + " (" + str(record.id) + ") to collection for " + user.username + ) except DatabaseError as de: logger.error( "Could not create record " - + basic_information.get('title') - + " (" + basic_information['id'] + ")\n" - + str(de)) + + basic_information.get("title") + + " (" + + basic_information["id"] + + ")\n" + + str(de) + ) else: - old_collection.remove(basic_information['id']) + old_collection.remove(basic_information["id"]) nr = nr + 1 if nr % 10 == 0: - progress.updateProgress('create', int((nr * 100) / tot)) - removed_records = UserRecords.objects.filter(user=user)\ - .filter(record_id__in=old_collection) + progress.updateProgress("create", int((nr * 100) / tot)) + removed_records = UserRecords.objects.filter(user=user).filter(record_id__in=old_collection) if removed_records.exists(): for ur in removed_records: - logger.info("Removed record " + ur.record.name - + " (" + str(ur.record.id) - + ") from collection for " + user.username) + logger.info( + "Removed record " + + ur.record.name + + " (" + + str(ur.record.id) + + ") from collection for " + + user.username + ) removed_records.delete() - progress.updateProgress('create', 100) + progress.updateProgress("create", 100) except discogs.DiscogsError as de: - logger.info("Did not find collection for " + user.username - + " on discogs\n" + str(de)) + logger.info("Did not find collection for " + user.username + " on discogs\n" + str(de)) return False return True diff --git a/records/services/rates.py b/records/services/rates.py index fddc370..d1df748 100644 --- a/records/services/rates.py +++ b/records/services/rates.py @@ -9,8 +9,8 @@ def getRate(currency): rates = cache.get(RATES_CACHE_KEY) yesterday = time.time() - 86400 - if not rates or rates.get('query').get('timestamp') < yesterday: - url = config('RATES_API_URL') + config('RATES_API_KEY') + if not rates or rates.get("query").get("timestamp") < yesterday: + url = config("RATES_API_URL") + config("RATES_API_KEY") r = requests.get(url) if r.status_code == 200: data = r.json() @@ -18,9 +18,9 @@ def getRate(currency): cache.set(RATES_CACHE_KEY, rates) else: return None - if currency == rates.get('query').get('base_currency'): + if currency == rates.get("query").get("base_currency"): return 1 - rate = rates.get('data').get(currency) + rate = rates.get("data").get(currency) if rate: return rate else: diff --git a/records/services/record.py b/records/services/record.py index 68b0ce0..b6f70c9 100644 --- a/records/services/record.py +++ b/records/services/record.py @@ -3,15 +3,7 @@ from django.core import files import requests from io import BytesIO -from ..models import ( - Record, - RecordArtists, - Listen, - RecordListens, - Track, - TrackArtists, - Artist -) +from ..models import Record, RecordArtists, Listen, RecordListens, Track, TrackArtists, Artist from .. import discogs from .. import spotify import records.services.artist as artistService @@ -23,40 +15,35 @@ def createRecord(id, data): release_id = id - master_id = data.get('master', release_id + 990000000) - if data.get('type') == "master": + master_id = data.get("master", release_id + 990000000) + if data.get("type") == "master": master_id = id try: release_id = Record.objects.filter(master=master_id)[0].id except IndexError: - release_id = data.get('main_release') + release_id = data.get("main_release") record, created = Record.objects.get_or_create( id=release_id, defaults={ - 'name': data.get('name'), - 'master': master_id, - 'cover': data.get('cover'), - 'thumbnail': data.get('thumbnail'), - 'year': data.get('year'), - 'format': __getFormat(data.get('format')) if data.get('format') else None - }) + "name": data.get("name"), + "master": master_id, + "cover": data.get("cover"), + "thumbnail": data.get("thumbnail"), + "year": data.get("year"), + "format": __getFormat(data.get("format")) if data.get("format") else None, + }, + ) if created: - logger.info("Created record " + record.name - + " (" + str(record.id) + ")") + logger.info("Created record " + record.name + " (" + str(record.id) + ")") position = 0 - for r_artist in data.get('artists'): + for r_artist in data.get("artists"): delimiter = None if type(r_artist) is Artist: artist = r_artist else: - artist = artistService.createArtist( - r_artist['id'], r_artist['name']) - delimiter = r_artist.get('join') - RecordArtists.objects.create( - record=record, - artist=artist, - delimiter=delimiter, - position=position) + artist = artistService.createArtist(r_artist["id"], r_artist["name"]) + delimiter = r_artist.get("join") + RecordArtists.objects.create(record=record, artist=artist, delimiter=delimiter, position=position) position += 1 return record @@ -66,110 +53,101 @@ def updateRecord(record): try: release_data = discogs.getRelease(record.id) __updateDataWithMasterData(release_data) - record.master = release_data.get('master_id') - __updateArtists(record, release_data.get('artists')) + record.master = release_data.get("master_id") + __updateArtists(record, release_data.get("artists")) record.track_set.all().delete() - if release_data.get('tracklist'): - for track_data in release_data.get('tracklist'): + if release_data.get("tracklist"): + for track_data in release_data.get("tracklist"): __createTrack(record, track_data) - if release_data.get('images'): + if release_data.get("images"): old_cover = record.cover - record.cover = release_data['images'][0].get('uri') - record.thumbnail = release_data['images'][0].get('uri150') + record.cover = release_data["images"][0].get("uri") + record.thumbnail = release_data["images"][0].get("uri150") if old_cover != record.cover: try: downloadCover(record) except requests.exceptions.RequestException as e: - logger.error("Error when downloading cover art for " - + record.name + " (" + str(record.id) + ")\n" - + str(e)) - __updateListens(record, release_data.get('videos')) - record.year = release_data.get('year') - if release_data.get('formats'): - record.format = __getFormat(release_data.get('formats')) - if release_data.get('lowest_price'): - record.price = release_data.get('lowest_price') + logger.error( + "Error when downloading cover art for " + record.name + " (" + str(record.id) + ")\n" + str(e) + ) + __updateListens(record, release_data.get("videos")) + record.year = release_data.get("year") + if release_data.get("formats"): + record.format = __getFormat(release_data.get("formats")) + if release_data.get("lowest_price"): + record.price = release_data.get("lowest_price") record.updated = date.today() record.save() except discogs.DiscogsError as de: - logger.info("Did not find record " + record.name - + " (" + str(record.id) + ") on discogs\n" + str(de)) + logger.info("Did not find record " + record.name + " (" + str(record.id) + ") on discogs\n" + str(de)) return False except DatabaseError as de: - logger.error("Could not update record " + record.name - + " (" + str(record.id) + ")\n" + str(de)) + logger.error("Could not update record " + record.name + " (" + str(record.id) + ")\n" + str(de)) return False return True def __updateDataWithMasterData(release_data): - if release_data.get('master_id'): + if release_data.get("master_id"): try: - master_data = discogs.getMaster(release_data.get('master_id')) - release_data['year'] = master_data.get('year') - if not release_data.get('images'): - if master_data.get('images'): - release_data['images'] = master_data.get('images') - if not release_data.get('videos'): - if master_data.get('videos'): - release_data['videos'] = master_data.get('videos') + master_data = discogs.getMaster(release_data.get("master_id")) + release_data["year"] = master_data.get("year") + if not release_data.get("images"): + if master_data.get("images"): + release_data["images"] = master_data.get("images") + if not release_data.get("videos"): + if master_data.get("videos"): + release_data["videos"] = master_data.get("videos") except discogs.DiscogsError as de: - logger.info("Did not find master for record " - + release_data.get('title') - + " (" + str(release_data.get('id')) + ") on discogs\n" - + str(de)) + logger.info( + "Did not find master for record " + + release_data.get("title") + + " (" + + str(release_data.get("id")) + + ") on discogs\n" + + str(de) + ) else: - release_data['master_id'] = release_data.get('id') + 990000000 + release_data["master_id"] = release_data.get("id") + 990000000 def __updateArtists(record, artists): RecordArtists.objects.filter(record=record).delete() position = 0 for r_artist in artists: - artist = artistService.createArtist( - r_artist['id'], r_artist['name']) - RecordArtists.objects.create( - record=record, - artist=artist, - delimiter=r_artist['join'], - position=position) + artist = artistService.createArtist(r_artist["id"], r_artist["name"]) + RecordArtists.objects.create(record=record, artist=artist, delimiter=r_artist["join"], position=position) position += 1 def __updateListens(record, videos): try: spotify_listen = Listen.objects.get(name="spotify") - if RecordListens.objects.filter(record=record, - listen=spotify_listen).count() == 0: + if RecordListens.objects.filter(record=record, listen=spotify_listen).count() == 0: try: - spotify_id = spotify.getAlbumId( - record.get_artist(), record.name) + spotify_id = spotify.getAlbumId(record.get_artist(), record.name) if spotify_id: - RecordListens.objects.create( - record=record, - listen=spotify_listen, - listen_key=spotify_id) + RecordListens.objects.create(record=record, listen=spotify_listen, listen_key=spotify_id) except spotify.SpotifyError as se: logger.error("Request to spotify failed:\n" + str(se)) except Listen.DoesNotExist: logger.error("Spotify listen does not exist") if videos: try: - youtube_listen = Listen.objects.get(name='youtube') - RecordListens.objects.filter( - record=record, listen=youtube_listen).delete() + youtube_listen = Listen.objects.get(name="youtube") + RecordListens.objects.filter(record=record, listen=youtube_listen).delete() for video in videos: - if "youtube" in video['uri'] and "v=" in video['uri']: - youtube_key = video['uri'][video['uri'].find('v=')+2:] - if RecordListens.objects.filter(record=record, - listen=youtube_listen, - listen_key=youtube_key)\ - .count() == 0: + if "youtube" in video["uri"] and "v=" in video["uri"]: + youtube_key = video["uri"][video["uri"].find("v=") + 2 :] + if ( + RecordListens.objects.filter( + record=record, listen=youtube_listen, listen_key=youtube_key + ).count() + == 0 + ): RecordListens.objects.create( - record=record, - listen=youtube_listen, - listen_key=youtube_key, - name=video.get('title')) + record=record, listen=youtube_listen, listen_key=youtube_key, name=video.get("title") + ) except Listen.DoesNotExist: logger.error("Youtube listen does not exist") @@ -177,9 +155,9 @@ def __updateListens(record, videos): def __getFormat(format_data): formats = [] for format in format_data: - format_string = format.get('name') - if format.get('descriptions'): - inch = re.search('(\\d+)"', " ".join(format.get('descriptions'))) + format_string = format.get("name") + if format.get("descriptions"): + inch = re.search('(\\d+)"', " ".join(format.get("descriptions"))) if inch: format_string += inch.group(1) formats.append(format_string.replace(" ", "-")) @@ -187,19 +165,12 @@ def __getFormat(format_data): def __createTrack(record, track_data): - track = Track.objects.create(position=track_data.get('position'), - name=track_data.get('title'), - record=record) - if track_data.get('artists'): + track = Track.objects.create(position=track_data.get("position"), name=track_data.get("title"), record=record) + if track_data.get("artists"): position = 0 - for t_artist in track_data.get('artists'): - artist = artistService.createArtist( - t_artist['id'], t_artist['name']) - TrackArtists.objects.create( - track=track, - artist=artist, - delimiter=t_artist['join'], - position=position) + for t_artist in track_data.get("artists"): + artist = artistService.createArtist(t_artist["id"], t_artist["name"]) + TrackArtists.objects.create(track=track, artist=artist, delimiter=t_artist["join"], position=position) position += 1 @@ -210,13 +181,12 @@ def downloadCover(record): record.cover = None return True headers = { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", } resp = requests.get(record.cover, headers=headers) resp.raise_for_status() fp = BytesIO() fp.write(resp.content) - record.cover_file.save(str(record.id) + ".jpg", files.File(fp)) + record.cover_file.save(str(record.id) + ".jpg", files.File(fp)) return False diff --git a/records/spotify.py b/records/spotify.py index 69d57ec..8bf2a14 100644 --- a/records/spotify.py +++ b/records/spotify.py @@ -23,26 +23,24 @@ def __str__(self): def getAlbum(artist, album): auth = "Bearer " + __getToken() headers = {"Authorization": auth} - query = "album:" + quote(album) + "%20artist:" + quote(artist) \ - + "&type=album" - r = requests.get(config('SPOTIFY_API_URL') - + "search?q=" + query, headers=headers) + query = "album:" + quote(album) + "%20artist:" + quote(artist) + "&type=album" + r = requests.get(config("SPOTIFY_API_URL") + "search?q=" + query, headers=headers) try: data = r.json() except JSONDecodeError: data = {} if r.status_code == 200: - if data['albums']['total'] > 0: - return data['albums']['items'][0] + if data["albums"]["total"] > 0: + return data["albums"]["items"][0] elif r.status_code >= 400: - raise SpotifyError(r.status_code, data['error'].get('message')) + raise SpotifyError(r.status_code, data["error"].get("message")) return None def getAlbumId(artist, album): album_data = getAlbum(artist, album) if album_data: - return album_data['id'] + return album_data["id"] return None @@ -50,22 +48,15 @@ def __getToken(): token_info = cache.get(TOKEN_KEY) if not token_info or time.time() > token_info[1]: token_info = __renewToken() - logger.debug("Renewed Spotify token, expires " - + time.asctime(time.localtime(token_info[1]))) + logger.debug("Renewed Spotify token, expires " + time.asctime(time.localtime(token_info[1]))) return token_info[0] def __renewToken(): - auth = base64.b64encode( - bytes(config('SPOTIFY_ID') + ":" + config('SPOTIFY_SECRET'), "utf-8")) - headers = { - "Authorization": "Basic " + auth.decode(), - "Content-Type": "application/x-www-form-urlencoded" - } + auth = base64.b64encode(bytes(config("SPOTIFY_ID") + ":" + config("SPOTIFY_SECRET"), "utf-8")) + headers = {"Authorization": "Basic " + auth.decode(), "Content-Type": "application/x-www-form-urlencoded"} data = {"grant_type": "client_credentials"} - r = requests.post(config('SPOTIFY_ACCOUNT_URL') - + "token", data=data, headers=headers) - token_info = (r.json()['access_token'], r.json() - ['expires_in'] + time.time()) + r = requests.post(config("SPOTIFY_ACCOUNT_URL") + "token", data=data, headers=headers) + token_info = (r.json()["access_token"], r.json()["expires_in"] + time.time()) cache.set(TOKEN_KEY, token_info) return token_info diff --git a/records/templates/records/index.html b/records/templates/records/index.html index 8559108..74e15f7 100644 --- a/records/templates/records/index.html +++ b/records/templates/records/index.html @@ -2,88 +2,97 @@ - + Sortera skivorna - - - + + + - - -
-

Skivorna

-
- Sök -
-
- - - -
-
0
- - + + +
+

Skivorna

+
Sök
+
+ + + +
+
0
+