Skip to content

Commit

Permalink
Sanitize input to rate endpoint. Lint all python files
Browse files Browse the repository at this point in the history
  • Loading branch information
GustavAndreasson committed Nov 9, 2023
1 parent ffe7efc commit fe02d94
Show file tree
Hide file tree
Showing 22 changed files with 529 additions and 592 deletions.
3 changes: 2 additions & 1 deletion frontend/urls.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from django.conf.urls import url
from . import views

urlpatterns = [
url(r'^$', views.index),
url(r"^$", views.index),
]
2 changes: 1 addition & 1 deletion frontend/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
def index(request):
if request.session.is_empty():
request.session.create() # Create session cookie if it does not exist
return render(request, 'frontend/index.html')
return render(request, "frontend/index.html")
58 changes: 27 additions & 31 deletions records/admin.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
from django.contrib import admin
from records.models import (
Record,
Artist,
RecordArtists,
Track,
Listen,
RecordListens,
ArtistMembers,
DiscogsUser
)
from records.models import Record, Artist, RecordArtists, Track, Listen, RecordListens, ArtistMembers, DiscogsUser
from records.services.record import updateRecord
from records.services.artist import updateArtist
from django.core.cache import cache

# Register your models here.


class ArtistInline(admin.TabularInline):
model = RecordArtists
readonly_fields = ('artist',)
readonly_fields = ("artist",)
extra = 0


Expand Down Expand Up @@ -45,12 +37,15 @@ def clear_cache_item(modeladmin, request, queryset):


class RecordAdmin(admin.ModelAdmin):
fields = ['id', 'name', 'master', 'year', 'format',
'cover', 'thumbnail', 'price', 'updated']
fields = ["id", "name", "master", "year", "format", "cover", "thumbnail", "price", "updated"]
inlines = [ArtistInline, TrackInline, ListenInline]
list_display = ('id', 'get_artist', 'name', 'format', 'updated')
search_fields = ['name']
actions = [reset_updated, update_record, clear_cache_item, ]
list_display = ("id", "get_artist", "name", "format", "updated")
search_fields = ["name"]
actions = [
reset_updated,
update_record,
clear_cache_item,
]


admin.site.register(Record, RecordAdmin)
Expand All @@ -63,43 +58,44 @@ def update_artist(modeladmin, request, queryset):

class MembersInline(admin.TabularInline):
model = ArtistMembers
readonly_fields = ('member',)
readonly_fields = ("member",)
extra = 0
fk_name = 'group'
fk_name = "group"


class GroupsInline(admin.TabularInline):
model = ArtistMembers
readonly_fields = ('group',)
readonly_fields = ("group",)
extra = 0
fk_name = 'member'
fk_name = "member"


class ArtistAdmin(admin.ModelAdmin):
fields = ['id', 'name', 'description',
'image', 'updated', 'collectionUpdated']
fields = ["id", "name", "description", "image", "updated", "collectionUpdated"]
inlines = [MembersInline, GroupsInline]
list_display = ('id', 'name', 'description', 'image',
'updated', 'collectionUpdated')
search_fields = ['name']
actions = [reset_updated, update_artist, ]
list_display = ("id", "name", "description", "image", "updated", "collectionUpdated")
search_fields = ["name"]
actions = [
reset_updated,
update_artist,
]


admin.site.register(Artist, ArtistAdmin)


class ListenAdmin(admin.ModelAdmin):
list_display = ('name', 'icon', 'template')
search_fields = ['name']
list_display = ("name", "icon", "template")
search_fields = ["name"]


admin.site.register(Listen, ListenAdmin)


class DiscogsUserAdmin(admin.ModelAdmin):
fields = ['username']
list_display = ('id', 'username')
search_fields = ['username']
fields = ["username"]
list_display = ("id", "username")
search_fields = ["username"]


admin.site.register(DiscogsUser, DiscogsUserAdmin)
2 changes: 1 addition & 1 deletion records/apps.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@


class RecordsConfig(AppConfig):
name = 'records'
name = "records"

def ready(self):
from . import signals
21 changes: 8 additions & 13 deletions records/management/commands/downloadcovers.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,28 @@ class Command(BaseCommand):
help = "Downloads covers to all records"

def add_arguments(self, parser):
parser.add_argument('limit', type=int,
help="Limit number of downloaded covers")
parser.add_argument("limit", type=int, help="Limit number of downloaded covers")

def handle(self, *args, **options):
records = Record.objects.filter(cover_file__exact='')[
:options['limit']]
records = Record.objects.filter(cover_file__exact="")[: options["limit"]]
for record in records:
self.stdout.write("Downloading cover for record " + record.name)
try:
if downloadCover(record):
record.save()
except HTTPError as e:
if e.response.status_code == 429:
self.stdout.write("Discogs download limit reached. Wait"
+ " for a minute before running again")
self.stdout.write("Discogs download limit reached. Wait" + " for a minute before running again")
break
elif e.response.status_code == 404:
self.stdout.write("Cover file for record " + record.name
+ " is missing on discogs. Updating "
+ "record.\n")
self.stdout.write(
"Cover file for record " + record.name + " is missing on discogs. Updating " + "record.\n"
)
updateRecord(record)
continue
else:
self.stdout.write("Error downloading cover for record "
+ record.name + "\n" + str(e))
self.stdout.write("Error downloading cover for record " + record.name + "\n" + str(e))
continue
except RequestException as e:
self.stdout.write("Error downloading cover for record "
+ record.name + "\n" + str(e))
self.stdout.write("Error downloading cover for record " + record.name + "\n" + str(e))
break
65 changes: 30 additions & 35 deletions records/management/commands/readartistxml.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,61 +8,58 @@ class Command(BaseCommand):
help = "Reads artist datadump xml and creates all artists"

def add_arguments(self, parser):
parser.add_argument('file', type=str,
help="XML file to import")
parser.add_argument('-s', '--start', type=int,
help="Position of first artist to import")
parser.add_argument('-e', '--end', type=int,
help="Position of last artist to import")
parser.add_argument("file", type=str, help="XML file to import")
parser.add_argument("-s", "--start", type=int, help="Position of first artist to import")
parser.add_argument("-e", "--end", type=int, help="Position of last artist to import")

def handle(self, *args, **options):
self.stdout.write("Counting number of artists in file...")
with open(options['file'], 'r') as f:
nr_artists = sum(line.count('<artist>') for line in f)
start = options['start'] if options['start'] else 0
end = min(options['end'], nr_artists) if options['end'] else nr_artists
with open(options["file"], "r") as f:
nr_artists = sum(line.count("<artist>") for line in f)
start = options["start"] if options["start"] else 0
end = min(options["end"], nr_artists) if options["end"] else nr_artists
if start > end:
self.stdout.write("No artists in range")
return
if options['start'] or options['end']:
self.stdout.write("Importing artists " + str(start) + " to "
+ str(end) + " of " + str(nr_artists)
+ " artists")
if options["start"] or options["end"]:
self.stdout.write(
"Importing artists " + str(start) + " to " + str(end) + " of " + str(nr_artists) + " artists"
)
nr_artists = end - start + 1
else:
self.stdout.write("Importing " + str(nr_artists) + " artists")
artist_counter = 0
artists_imported = 0
loop_counter = 0
batch_size = 1000
context = etree.iterparse(
options['file'], events=('end',), tag='artist')
context = etree.iterparse(options["file"], events=("end",), tag="artist")
artists_batch = []
for event, elem in context:
if loop_counter >= start:
if elem.find('name').text and elem.find('id').text and \
not elem.find('data_quality').text in \
('Needs Major Changes', 'Entirely Incorrect'):
name = fixArtistName(elem.find('name').text)
if (
elem.find("name").text
and elem.find("id").text
and not elem.find("data_quality").text in ("Needs Major Changes", "Entirely Incorrect")
):
name = fixArtistName(elem.find("name").text)
sname = name[:20].lower()
artists_batch.append(Artist(id=elem.find('id').text,
name=name,
sname=sname))
artists_batch.append(Artist(id=elem.find("id").text, name=name, sname=sname))
if len(artists_batch) >= batch_size:
Artist.objects.bulk_create(artists_batch,
ignore_conflicts=True)
Artist.objects.bulk_create(artists_batch, ignore_conflicts=True)
artists_batch = []
artists_imported += batch_size
artist_counter += 1
if artist_counter % 863 == 1 or artist_counter == nr_artists:
progress = (artist_counter / nr_artists) * 100
self.stdout.write(
"[{0}] {1}% {2}/{3}".format('#'*int(progress/2)
+ ' '*(50-int(progress/2)),
int(progress),
artist_counter,
nr_artists),
ending='\r')
"[{0}] {1}% {2}/{3}".format(
"#" * int(progress / 2) + " " * (50 - int(progress / 2)),
int(progress),
artist_counter,
nr_artists,
),
ending="\r",
)
self.stdout.flush()
elem.clear()
while elem.getprevious() is not None:
Expand All @@ -71,8 +68,6 @@ def handle(self, *args, **options):
if loop_counter > end:
break
if artists_batch:
Artist.objects.bulk_create(artists_batch,
ignore_conflicts=True)
Artist.objects.bulk_create(artists_batch, ignore_conflicts=True)
artists_imported += len(artists_batch)
self.stdout.write("\nArtist import done. " + str(artists_imported)
+ " artists imported.")
self.stdout.write("\nArtist import done. " + str(artists_imported) + " artists imported.")
6 changes: 2 additions & 4 deletions records/management/commands/updaterecords.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,10 @@ class Command(BaseCommand):
help = "Loads extra info to all records"

def add_arguments(self, parser):
parser.add_argument('limit', type=int,
help="Limit number of updated records")
parser.add_argument("limit", type=int, help="Limit number of updated records")

def handle(self, *args, **options):
records = Record.objects.filter(updated__isnull=True)[
:options['limit']]
records = Record.objects.filter(updated__isnull=True)[: options["limit"]]
for record in records:
self.stdout.write("Updating record " + record.name)
updateRecord(record)
Loading

0 comments on commit fe02d94

Please sign in to comment.