Initial PoC with contribution import from Gitlab issues

.dockerignore

@@ -2,3 +2,4 @@
 !.coveragerc
 !.env
 !.pylintrc
+*.pyc

config/urls.py

@@ -5,23 +5,7 @@ from django.contrib import admin
 from django.views.generic import TemplateView
 from django.views import defaults as default_views
 
-urlpatterns = [
-    path("", TemplateView.as_view(template_name="pages/home.html"), name="home"),
-    path(
-        "about/",
-        TemplateView.as_view(template_name="pages/about.html"),
-        name="about",
-    ),
-    # Django Admin, use {% url 'admin:index' %}
-    path(settings.ADMIN_URL, admin.site.urls),
-    # User management
-    path(
-        "users/",
-        include("contributions.users.urls", namespace="users"),
-    ),
-    path("accounts/", include("allauth.urls")),
-    # Your stuff: custom urls includes go here
-] + static(
+urlpatterns = [path(settings.ADMIN_URL, admin.site.urls)] + static(
     settings.MEDIA_URL, document_root=settings.MEDIA_ROOT
 )
 

contributions/core/admin.py

+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.Contributor)
+class ContributorAdmin(admin.ModelAdmin):
+    list_display = ["username", "name", "creation_date"]
+    search_fields = ["username", "name"]
+
+
+@admin.register(models.Contribution)
+class ContributionAdmin(admin.ModelAdmin):
+    list_display = [
+        "contributor",
+        "summary",
+        "creation_date",
+        "import_date",
+        "type",
+        "external_id",
+    ]
+    search_fields = [
+        "contributor__username",
+        "contributor__name",
+        "summary",
+        "external_id",
+    ]
+
+    list_filter = ["contributor", "type"]
+
+    list_select_related = ["contributor"]

contributions/core/migrations/0001_initial.py

+# Generated by Django 2.1.2 on 2018-10-06 15:48
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Contribution',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('summary', models.CharField(max_length=500)),
+                ('type', models.CharField(choices=[('dev', 'Development'), ('i18n', 'Translations'), ('network', 'Network'), ('donation', 'Donations'), ('communication', 'Communication'), ('other', 'Other')], max_length=50)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('import_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('is_visible', models.BooleanField(default=True)),
+                ('external_id', models.CharField(max_length=250, unique=True)),
+                ('url', models.URLField()),
+                ('metadata', django.contrib.postgres.fields.jsonb.JSONField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Contributor',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('username', models.CharField(max_length=200, unique=True)),
+                ('name', models.CharField(max_length=200)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('is_visible', models.BooleanField(default=True)),
+                ('metadata', django.contrib.postgres.fields.jsonb.JSONField()),
+            ],
+        ),
+        migrations.AddField(
+            model_name='contribution',
+            name='contributor',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='contributions', to='core.Contributor'),
+        ),
+    ]

contributions/core/models.py

+from django.contrib.postgres.fields import JSONField
+from django.db import models
+from django.utils import timezone
+
+
+class Contributor(models.Model):
+    username = models.CharField(max_length=200, unique=True)
+    name = models.CharField(max_length=200)
+    creation_date = models.DateTimeField(default=timezone.now)
+    is_visible = models.BooleanField(default=True)
+    metadata = JSONField()
+
+    class Meta:
+        ordering = ("-creation_date",)
+
+    def __str__(self):
+        return self.username
+
+
+CONTRIBUTION_TYPES = [
+    ("dev", "Development"),
+    ("i18n", "Translations"),
+    ("network", "Network"),
+    ("donation", "Donations"),
+    ("communication", "Communication"),
+    ("other", "Other"),
+]
+
+
+class Contribution(models.Model):
+    contributor = models.ForeignKey(
+        Contributor, related_name="contributions", on_delete=models.CASCADE
+    )
+    summary = models.CharField(max_length=500)
+    type = models.CharField(max_length=50, choices=CONTRIBUTION_TYPES)
+    creation_date = models.DateTimeField(default=timezone.now)
+    import_date = models.DateTimeField(default=timezone.now)
+    is_visible = models.BooleanField(default=True)
+    external_id = models.CharField(max_length=250, unique=True)
+    url = models.URLField()
+    metadata = JSONField()
+
+    class Meta:
+        ordering = ("-creation_date",)

contributions/sources/gitlab.py

+import requests
+import pendulum
+
+from django.utils import timezone
+
+from contributions.core import models as core_models
+
+
+def retrieve_issue(gitlab_url, project_id, issue_id):
+    url = f"{gitlab_url}/api/v4/projects/{project_id}/issues/{issue_id}"
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.json()
+
+
+def retrieve_issue_page(url):
+    response = requests.get(url)
+    response.raise_for_status()
+
+    links = response.links
+    next_page = None
+    if links and links.get("next"):
+        next_page = links["next"]["url"]
+    return response.json(), next_page
+
+
+def import_contributor_from_author(payload):
+    contributor = None
+    try:
+        contributor = core_models.Contributor.objects.get(username=payload["username"])
+    except core_models.Contributor.DoesNotExist:
+        return core_models.Contributor.objects.create(
+            username=payload["username"],
+            name=payload["name"],
+            metadata={"gitlab": payload},
+        )
+
+    contributor.metadata["gitlab"] = payload
+    contributor.save(update_fields=["metadata"])
+    return contributor
+
+
+def import_issue_as_contribution(payload):
+    contributor = import_contributor_from_author(payload["author"])
+    external_id = payload["_links"]["self"]
+    defaults = {
+        "summary": payload["title"],
+        "contributor": contributor,
+        "type": "dev",
+        "creation_date": pendulum.parse(payload["created_at"]),
+        "import_date": timezone.now(),
+        "is_visible": True,
+        "url": payload["web_url"],
+        "metadata": {"labels": payload["labels"]},
+    }
+
+    return core_models.Contribution.objects.update_or_create(
+        external_id=external_id, defaults=defaults
+    )[0]

local.yml

 version: '3'
 
-volumes:
-  local_postgres_data: {}
-  local_postgres_data_backups: {}
-
 services:
   django:
     build:
@@ -27,7 +23,6 @@ services:
       dockerfile: ./compose/production/postgres/Dockerfile
     image: contributions_production_postgres
     volumes:
-      - local_postgres_data:/var/lib/postgresql/data
-      - local_postgres_data_backups:/backups
+      - ./data/postgres:/var/lib/postgresql/data
     env_file:
       - ./.envs/.local/.postgres

pytest.ini

 [pytest]
 DJANGO_SETTINGS_MODULE=config.settings.test
+testpaths = tests

requirements/base.txt

@@ -12,3 +12,5 @@ django-redis==4.9.0  # https://github.com/niwinz/django-redis
 # Django REST Framework
 djangorestframework>=3.8,<3.9  # https://github.com/encode/django-rest-framework
 psycopg2==2.7.4 --no-binary psycopg2  # https://github.com/psycopg/psycopg2
+requests
+pendulum

requirements/local.txt

@@ -23,3 +23,5 @@ django-debug-toolbar==1.10.1  # https://github.com/jazzband/django-debug-toolbar
 django-extensions==2.1.3  # https://github.com/django-extensions/django-extensions
 django-coverage-plugin==1.6.0  # https://github.com/nedbat/django_coverage_plugin
 pytest-django==3.4.3  # https://github.com/pytest-dev/pytest-django
+requests-mock==1.5.2
+pytest-mock==1.10.0

tests/conftest.py

 import pytest
 from django.conf import settings
 from django.test import RequestFactory
+from django.utils import timezone
 
-
-@pytest.fixture(autouse=True)
-def media_storage(settings, tmpdir):
-    settings.MEDIA_ROOT = tmpdir.strpath
+from . import factories as test_factories
 
 
 @pytest.fixture
 def request_factory() -> RequestFactory:
     return RequestFactory()
+
+
+@pytest.fixture
+def factories(db) -> test_factories:
+    return test_factories
+
+
+@pytest.fixture
+def nodb_factories() -> test_factories:
+    return test_factories
+
+
+@pytest.fixture
+def now(mocker):
+    n = timezone.now()
+    mocker.patch.object(timezone, 'now', return_value=n)
+    return n

tests/factories.py

+import factory
+
+from contributions.core import models as core_models
+
+
+class Contributor(factory.DjangoModelFactory):
+    name = factory.Faker('name')
+    metadata = factory.LazyAttribute(lambda o: {})
+
+    class Meta:
+        model = core_models.Contributor
+
+
+class Contribution(factory.DjangoModelFactory):
+    contributor = factory.SubFactory(Contributor)
+    summary = factory.Faker('paragraph')
+    type = factory.Iterator([t[0] for t in core_models.CONTRIBUTION_TYPES])
+    external_id = factory.Faker('uuid4')
+    url = factory.Faker('url')
+    metadata = factory.LazyAttribute(lambda o: {})
+
+    class Meta:
+        model = core_models.Contribution

tests/test_gitlab.py

+from contributions.sources import gitlab
+import pendulum
+
+
+def test_retrieve_issue(requests_mock):
+    gitlab_url = "https://gitlab.test"
+    project_id = 1
+    issue_id = 42
+
+    requests_mock.get(
+        f"https://gitlab.test/api/v4/projects/{project_id}/issues/{issue_id}",
+        json={"hello": "world"},
+    )
+    result = gitlab.retrieve_issue(gitlab_url, project_id, issue_id)
+
+    assert result == {"hello": "world"}
+
+
+def test_import_issue_as_contribution(now, db):
+    issue_payload = {
+        "id": 703,
+        "iid": 559,
+        "project_id": 17,
+        "title": "My awesome issue",
+        "created_at": "2018-10-05T15:23:17.257Z",
+        "labels": ["tag1", "tag2", "tag3"],
+        "author": {
+            "id": 130,
+            "name": "Alice",
+            "username": "Alice42",
+            "avatar_url": "https://secure.gravatar.com/avatar/1ed9cb8b4bdad28157104cd2f9468b3f?s=80\u0026d=identicon",
+            "web_url": "https://gitlab.test/Alice42",
+        },
+        "web_url": "https://gitlab.test/awesome/project/issues/559",
+        "_links": {"self": "https://gitlab.test/api/v4/projects/17/issues/559"},
+    }
+
+    contribution = gitlab.import_issue_as_contribution(issue_payload)
+
+    contributor = contribution.contributor
+
+    assert contributor.name == issue_payload["author"]["name"]
+    assert contributor.username == issue_payload["author"]["username"]
+    assert contributor.metadata == {"gitlab": issue_payload["author"]}
+
+    assert contribution.summary == issue_payload["title"]
+    assert contribution.type == "dev"
+    assert contribution.creation_date == pendulum.parse(issue_payload["created_at"])
+    assert contribution.import_date == now
+    assert contribution.is_visible is True
+    assert contribution.external_id == issue_payload["_links"]["self"]
+    assert contribution.url == issue_payload["web_url"]
+    assert contribution.metadata == {"labels": issue_payload["labels"]}
+
+
+def test_import_contributor_create(db):
+    payload = {"id": 130, "name": "Alice", "username": "Alice42"}
+
+    contributor = gitlab.import_contributor_from_author(payload)
+
+    assert contributor.name == payload["name"]
+    assert contributor.username == payload["username"]
+    assert contributor.metadata == {"gitlab": payload}
+
+
+def test_import_contributor_update_metadata(factories):
+    existing = factories.Contributor(username="Alice42", metadata={"hello": "world"})
+    payload = {"id": 130, "name": "Alice", "username": "Alice42"}
+
+    contributor = gitlab.import_contributor_from_author(payload)
+    contributor == existing
+
+    assert contributor.name == existing.name
+    assert contributor.username == existing.username
+    assert contributor.metadata == {"gitlab": payload, "hello": "world"}
+
+
+def test_retrieve_issue_page(requests_mock):
+    gitlab_url = "https://gitlab.test"
+    project_id = 1
+    url = f"{gitlab_url}/api/v4/projects/{project_id}/issues"
+
+    expected_next_page = (
+        f"https://gitlab.test/api/v4/projects/{project_id}/issues?page=2"
+    )
+    requests_mock.get(
+        f"https://gitlab.test/api/v4/projects/{project_id}/issues",
+        json=["hello", "world"],
+        headers={"Link": f'<{expected_next_page}>; rel="next"'},
+    )
+    result, next_page = gitlab.retrieve_issue_page(url)
+
+    assert result == ["hello", "world"]
+    assert expected_next_page == next_page

tests/test_models.py

 from django.utils import timezone
 
-from . import factories
+from contributions.core import models as core_models
 
 
-def test_contributor_factory():
-    c = factories.Contributor(
-        name='alice',
+def test_contributor_factory(factories):
+    contributor = factories.Contributor(
+        name="alice", is_visible=True, metadata={"hello": "world"}
+    )
+
+    assert contributor.name == "alice"
+    assert contributor.creation_date < timezone.now()
+    assert contributor.is_visible is True
+    assert contributor.metadata == {"hello": "world"}
+
+
+def test_contribution_factory(factories):
+    contribution = factories.Contribution(
+        external_id="weblate:front",
+        metadata={"locale": "en"},
         is_visible=True,
-        links={
-            'hello': 'world'
-        }
+        url="https://weblate.test",
+        summary="Hello",
+        type="i18n:translation",
     )
 
-    assert c.name == 'alice'
-    assert c.creation_date < timezone.now()
-    assert c.is_visible is True
-    assert c.links == {
-        'hello': 'world'
-    }
+    assert isinstance(contribution.contributor, core_models.Contributor)
+    assert contribution.creation_date < timezone.now()
+    assert contribution.import_date < timezone.now()
+    assert contribution.is_visible is True
+    assert contribution.metadata == {"locale": "en"}
+    assert contribution.summary == "Hello"
+    assert contribution.type == "i18n:translation"
+    assert contribution.external_id == "weblate:front"
+    assert contribution.url == "https://weblate.test"