Verified Commit 4781e782 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Fix #376: Smarter date parsing during import by replacing arrow with pendulum

parent 8974881f
import arrow import datetime
import mutagen import mutagen
import pendulum
from django import forms from django import forms
NODEFAULT = object() NODEFAULT = object()
...@@ -101,6 +102,11 @@ class FirstUUIDField(forms.UUIDField): ...@@ -101,6 +102,11 @@ class FirstUUIDField(forms.UUIDField):
return super().to_python(value) return super().to_python(value)
def get_date(value):
parsed = pendulum.parse(str(value))
return datetime.date(parsed.year, parsed.month, parsed.day)
VALIDATION = { VALIDATION = {
"musicbrainz_artistid": FirstUUIDField(), "musicbrainz_artistid": FirstUUIDField(),
"musicbrainz_albumid": FirstUUIDField(), "musicbrainz_albumid": FirstUUIDField(),
...@@ -118,7 +124,7 @@ CONF = { ...@@ -118,7 +124,7 @@ CONF = {
"title": {}, "title": {},
"artist": {}, "artist": {},
"album": {}, "album": {},
"date": {"field": "date", "to_application": lambda v: arrow.get(v).date()}, "date": {"field": "date", "to_application": get_date},
"musicbrainz_albumid": {}, "musicbrainz_albumid": {},
"musicbrainz_artistid": {}, "musicbrainz_artistid": {},
"musicbrainz_recordingid": {"field": "musicbrainz_trackid"}, "musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
...@@ -134,7 +140,7 @@ CONF = { ...@@ -134,7 +140,7 @@ CONF = {
"title": {}, "title": {},
"artist": {}, "artist": {},
"album": {}, "album": {},
"date": {"field": "date", "to_application": lambda v: arrow.get(v).date()}, "date": {"field": "date", "to_application": get_date},
"musicbrainz_albumid": {"field": "MusicBrainz Album Id"}, "musicbrainz_albumid": {"field": "MusicBrainz Album Id"},
"musicbrainz_artistid": {"field": "MusicBrainz Artist Id"}, "musicbrainz_artistid": {"field": "MusicBrainz Artist Id"},
"musicbrainz_recordingid": {"field": "MusicBrainz Track Id"}, "musicbrainz_recordingid": {"field": "MusicBrainz Track Id"},
...@@ -148,10 +154,7 @@ CONF = { ...@@ -148,10 +154,7 @@ CONF = {
"title": {"field": "TIT2"}, "title": {"field": "TIT2"},
"artist": {"field": "TPE1"}, "artist": {"field": "TPE1"},
"album": {"field": "TALB"}, "album": {"field": "TALB"},
"date": { "date": {"field": "TDRC", "to_application": get_date},
"field": "TDRC",
"to_application": lambda v: arrow.get(str(v)).date(),
},
"musicbrainz_albumid": {"field": "MusicBrainz Album Id"}, "musicbrainz_albumid": {"field": "MusicBrainz Album Id"},
"musicbrainz_artistid": {"field": "MusicBrainz Artist Id"}, "musicbrainz_artistid": {"field": "MusicBrainz Artist Id"},
"musicbrainz_recordingid": { "musicbrainz_recordingid": {
...@@ -172,10 +175,7 @@ CONF = { ...@@ -172,10 +175,7 @@ CONF = {
"title": {}, "title": {},
"artist": {}, "artist": {},
"album": {}, "album": {},
"date": { "date": {"field": "date", "to_application": get_date},
"field": "date",
"to_application": lambda v: arrow.get(str(v)).date(),
},
"musicbrainz_albumid": {}, "musicbrainz_albumid": {},
"musicbrainz_artistid": {}, "musicbrainz_artistid": {},
"musicbrainz_recordingid": {"field": "musicbrainz_trackid"}, "musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
......
import datetime
import os import os
import shutil import shutil
import tempfile import tempfile
import uuid import uuid
import arrow
import markdown import markdown
import pendulum
from django.conf import settings from django.conf import settings
from django.core.files import File from django.core.files import File
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
...@@ -125,9 +124,7 @@ def import_artist(v): ...@@ -125,9 +124,7 @@ def import_artist(v):
def parse_date(v): def parse_date(v):
if len(v) == 4: d = pendulum.parse(v).date()
return datetime.date(int(v), 1, 1)
d = arrow.get(v).date()
return d return d
......
...@@ -35,7 +35,7 @@ djangorestframework>=3.7,<3.8 ...@@ -35,7 +35,7 @@ djangorestframework>=3.7,<3.8
djangorestframework-jwt>=1.11,<1.12 djangorestframework-jwt>=1.11,<1.12
oauth2client<4 oauth2client<4
google-api-python-client>=1.6,<1.7 google-api-python-client>=1.6,<1.7
arrow>=0.12,<0.13 pendulum>=2,<3
persisting-theory>=0.2,<0.3 persisting-theory>=0.2,<0.3
django-versatileimagefield>=1.9,<1.10 django-versatileimagefield>=1.9,<1.10
django-filter>=1.1,<1.2 django-filter>=1.1,<1.2
......
import arrow import pendulum
import pytest import pytest
from django.urls import reverse from django.urls import reverse
from django.utils import timezone from django.utils import timezone
...@@ -455,7 +455,7 @@ def test_library_actor_handle_create_audio(mocker, factories): ...@@ -455,7 +455,7 @@ def test_library_actor_handle_create_audio(mocker, factories):
assert lt.title == a["metadata"]["recording"]["title"] assert lt.title == a["metadata"]["recording"]["title"]
assert lt.artist_name == a["metadata"]["artist"]["name"] assert lt.artist_name == a["metadata"]["artist"]["name"]
assert lt.album_title == a["metadata"]["release"]["title"] assert lt.album_title == a["metadata"]["release"]["title"]
assert lt.published_date == arrow.get(a["published"]) assert lt.published_date == pendulum.parse(a["published"])
def test_library_actor_handle_create_audio_autoimport(mocker, factories): def test_library_actor_handle_create_audio_autoimport(mocker, factories):
...@@ -494,7 +494,7 @@ def test_library_actor_handle_create_audio_autoimport(mocker, factories): ...@@ -494,7 +494,7 @@ def test_library_actor_handle_create_audio_autoimport(mocker, factories):
assert lt.title == a["metadata"]["recording"]["title"] assert lt.title == a["metadata"]["recording"]["title"]
assert lt.artist_name == a["metadata"]["artist"]["name"] assert lt.artist_name == a["metadata"]["artist"]["name"]
assert lt.album_title == a["metadata"]["release"]["title"] assert lt.album_title == a["metadata"]["release"]["title"]
assert lt.published_date == arrow.get(a["published"]) assert lt.published_date == pendulum.parse(a["published"])
batch = music_models.ImportBatch.objects.latest("id") batch = music_models.ImportBatch.objects.latest("id")
......
import arrow import pendulum
import pytest import pytest
from django.core.paginator import Paginator from django.core.paginator import Paginator
...@@ -492,7 +492,7 @@ def test_activity_pub_audio_serializer_to_library_track(factories): ...@@ -492,7 +492,7 @@ def test_activity_pub_audio_serializer_to_library_track(factories):
assert lt.title == audio["metadata"]["recording"]["title"] assert lt.title == audio["metadata"]["recording"]["title"]
assert lt.artist_name == audio["metadata"]["artist"]["name"] assert lt.artist_name == audio["metadata"]["artist"]["name"]
assert lt.album_title == audio["metadata"]["release"]["title"] assert lt.album_title == audio["metadata"]["release"]["title"]
assert lt.published_date == arrow.get(audio["published"]) assert lt.published_date == pendulum.parse(audio["published"])
def test_activity_pub_audio_serializer_to_library_track_no_duplicate(factories): def test_activity_pub_audio_serializer_to_library_track_no_duplicate(factories):
......
...@@ -122,3 +122,11 @@ def test_mbid_clean_keeps_only_first(field_name): ...@@ -122,3 +122,11 @@ def test_mbid_clean_keeps_only_first(field_name):
result = field.to_python("/".join([u1, u2])) result = field.to_python("/".join([u1, u2]))
assert str(result) == u1 assert str(result) == u1
@pytest.mark.parametrize(
"raw,expected",
[("2017", datetime.date(2017, 1, 1)), ("2017-12-31", datetime.date(2017, 12, 31))],
)
def test_date_parsing(raw, expected):
assert metadata.get_date(raw) == expected
Smarter date parsing during import by replacing arrow with pendulum (#376)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment