Verified Commit 3f31a79a authored by Agate's avatar Agate 💬

Initial PoC with contribution import from Gitlab issues

parent d047e643
......@@ -2,3 +2,4 @@
!.coveragerc
!.env
!.pylintrc
*.pyc
This diff is collapsed.
......@@ -5,23 +5,7 @@ from django.contrib import admin
from django.views.generic import TemplateView
from django.views import defaults as default_views
urlpatterns = [
path("", TemplateView.as_view(template_name="pages/home.html"), name="home"),
path(
"about/",
TemplateView.as_view(template_name="pages/about.html"),
name="about",
),
# Django Admin, use {% url 'admin:index' %}
path(settings.ADMIN_URL, admin.site.urls),
# User management
path(
"users/",
include("contributions.users.urls", namespace="users"),
),
path("accounts/", include("allauth.urls")),
# Your stuff: custom urls includes go here
] + static(
urlpatterns = [path(settings.ADMIN_URL, admin.site.urls)] + static(
settings.MEDIA_URL, document_root=settings.MEDIA_ROOT
)
......
from django.contrib import admin
from . import models
@admin.register(models.Contributor)
class ContributorAdmin(admin.ModelAdmin):
list_display = ["username", "name", "creation_date"]
search_fields = ["username", "name"]
@admin.register(models.Contribution)
class ContributionAdmin(admin.ModelAdmin):
list_display = [
"contributor",
"summary",
"creation_date",
"import_date",
"type",
"external_id",
]
search_fields = [
"contributor__username",
"contributor__name",
"summary",
"external_id",
]
list_filter = ["contributor", "type"]
list_select_related = ["contributor"]
# Generated by Django 2.1.2 on 2018-10-06 15:48
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Contribution',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('summary', models.CharField(max_length=500)),
('type', models.CharField(choices=[('dev', 'Development'), ('i18n', 'Translations'), ('network', 'Network'), ('donation', 'Donations'), ('communication', 'Communication'), ('other', 'Other')], max_length=50)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('import_date', models.DateTimeField(default=django.utils.timezone.now)),
('is_visible', models.BooleanField(default=True)),
('external_id', models.CharField(max_length=250, unique=True)),
('url', models.URLField()),
('metadata', django.contrib.postgres.fields.jsonb.JSONField()),
],
),
migrations.CreateModel(
name='Contributor',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('username', models.CharField(max_length=200, unique=True)),
('name', models.CharField(max_length=200)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('is_visible', models.BooleanField(default=True)),
('metadata', django.contrib.postgres.fields.jsonb.JSONField()),
],
),
migrations.AddField(
model_name='contribution',
name='contributor',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='contributions', to='core.Contributor'),
),
]
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils import timezone
class Contributor(models.Model):
username = models.CharField(max_length=200, unique=True)
name = models.CharField(max_length=200)
creation_date = models.DateTimeField(default=timezone.now)
is_visible = models.BooleanField(default=True)
metadata = JSONField()
class Meta:
ordering = ("-creation_date",)
def __str__(self):
return self.username
CONTRIBUTION_TYPES = [
("dev", "Development"),
("i18n", "Translations"),
("network", "Network"),
("donation", "Donations"),
("communication", "Communication"),
("other", "Other"),
]
class Contribution(models.Model):
contributor = models.ForeignKey(
Contributor, related_name="contributions", on_delete=models.CASCADE
)
summary = models.CharField(max_length=500)
type = models.CharField(max_length=50, choices=CONTRIBUTION_TYPES)
creation_date = models.DateTimeField(default=timezone.now)
import_date = models.DateTimeField(default=timezone.now)
is_visible = models.BooleanField(default=True)
external_id = models.CharField(max_length=250, unique=True)
url = models.URLField()
metadata = JSONField()
class Meta:
ordering = ("-creation_date",)
import requests
import pendulum
from django.utils import timezone
from contributions.core import models as core_models
def retrieve_issue(gitlab_url, project_id, issue_id):
url = f"{gitlab_url}/api/v4/projects/{project_id}/issues/{issue_id}"
response = requests.get(url)
response.raise_for_status()
return response.json()
def retrieve_issue_page(url):
response = requests.get(url)
response.raise_for_status()
links = response.links
next_page = None
if links and links.get("next"):
next_page = links["next"]["url"]
return response.json(), next_page
def import_contributor_from_author(payload):
contributor = None
try:
contributor = core_models.Contributor.objects.get(username=payload["username"])
except core_models.Contributor.DoesNotExist:
return core_models.Contributor.objects.create(
username=payload["username"],
name=payload["name"],
metadata={"gitlab": payload},
)
contributor.metadata["gitlab"] = payload
contributor.save(update_fields=["metadata"])
return contributor
def import_issue_as_contribution(payload):
contributor = import_contributor_from_author(payload["author"])
external_id = payload["_links"]["self"]
defaults = {
"summary": payload["title"],
"contributor": contributor,
"type": "dev",
"creation_date": pendulum.parse(payload["created_at"]),
"import_date": timezone.now(),
"is_visible": True,
"url": payload["web_url"],
"metadata": {"labels": payload["labels"]},
}
return core_models.Contribution.objects.update_or_create(
external_id=external_id, defaults=defaults
)[0]
version: '3'
volumes:
local_postgres_data: {}
local_postgres_data_backups: {}
services:
django:
build:
......@@ -27,7 +23,6 @@ services:
dockerfile: ./compose/production/postgres/Dockerfile
image: contributions_production_postgres
volumes:
- local_postgres_data:/var/lib/postgresql/data
- local_postgres_data_backups:/backups
- ./data/postgres:/var/lib/postgresql/data
env_file:
- ./.envs/.local/.postgres
[pytest]
DJANGO_SETTINGS_MODULE=config.settings.test
testpaths = tests
......@@ -12,3 +12,5 @@ django-redis==4.9.0 # https://github.com/niwinz/django-redis
# Django REST Framework
djangorestframework>=3.8,<3.9 # https://github.com/encode/django-rest-framework
psycopg2==2.7.4 --no-binary psycopg2 # https://github.com/psycopg/psycopg2
requests
pendulum
......@@ -23,3 +23,5 @@ django-debug-toolbar==1.10.1 # https://github.com/jazzband/django-debug-toolbar
django-extensions==2.1.3 # https://github.com/django-extensions/django-extensions
django-coverage-plugin==1.6.0 # https://github.com/nedbat/django_coverage_plugin
pytest-django==3.4.3 # https://github.com/pytest-dev/pytest-django
requests-mock==1.5.2
pytest-mock==1.10.0
import pytest
from django.conf import settings
from django.test import RequestFactory
from django.utils import timezone
@pytest.fixture(autouse=True)
def media_storage(settings, tmpdir):
settings.MEDIA_ROOT = tmpdir.strpath
from . import factories as test_factories
@pytest.fixture
def request_factory() -> RequestFactory:
return RequestFactory()
@pytest.fixture
def factories(db) -> test_factories:
return test_factories
@pytest.fixture
def nodb_factories() -> test_factories:
return test_factories
@pytest.fixture
def now(mocker):
n = timezone.now()
mocker.patch.object(timezone, 'now', return_value=n)
return n
import factory
from contributions.core import models as core_models
class Contributor(factory.DjangoModelFactory):
name = factory.Faker('name')
metadata = factory.LazyAttribute(lambda o: {})
class Meta:
model = core_models.Contributor
class Contribution(factory.DjangoModelFactory):
contributor = factory.SubFactory(Contributor)
summary = factory.Faker('paragraph')
type = factory.Iterator([t[0] for t in core_models.CONTRIBUTION_TYPES])
external_id = factory.Faker('uuid4')
url = factory.Faker('url')
metadata = factory.LazyAttribute(lambda o: {})
class Meta:
model = core_models.Contribution
from contributions.sources import gitlab
import pendulum
def test_retrieve_issue(requests_mock):
gitlab_url = "https://gitlab.test"
project_id = 1
issue_id = 42
requests_mock.get(
f"https://gitlab.test/api/v4/projects/{project_id}/issues/{issue_id}",
json={"hello": "world"},
)
result = gitlab.retrieve_issue(gitlab_url, project_id, issue_id)
assert result == {"hello": "world"}
def test_import_issue_as_contribution(now, db):
issue_payload = {
"id": 703,
"iid": 559,
"project_id": 17,
"title": "My awesome issue",
"created_at": "2018-10-05T15:23:17.257Z",
"labels": ["tag1", "tag2", "tag3"],
"author": {
"id": 130,
"name": "Alice",
"username": "Alice42",
"avatar_url": "https://secure.gravatar.com/avatar/1ed9cb8b4bdad28157104cd2f9468b3f?s=80\u0026d=identicon",
"web_url": "https://gitlab.test/Alice42",
},
"web_url": "https://gitlab.test/awesome/project/issues/559",
"_links": {"self": "https://gitlab.test/api/v4/projects/17/issues/559"},
}
contribution = gitlab.import_issue_as_contribution(issue_payload)
contributor = contribution.contributor
assert contributor.name == issue_payload["author"]["name"]
assert contributor.username == issue_payload["author"]["username"]
assert contributor.metadata == {"gitlab": issue_payload["author"]}
assert contribution.summary == issue_payload["title"]
assert contribution.type == "dev"
assert contribution.creation_date == pendulum.parse(issue_payload["created_at"])
assert contribution.import_date == now
assert contribution.is_visible is True
assert contribution.external_id == issue_payload["_links"]["self"]
assert contribution.url == issue_payload["web_url"]
assert contribution.metadata == {"labels": issue_payload["labels"]}
def test_import_contributor_create(db):
payload = {"id": 130, "name": "Alice", "username": "Alice42"}
contributor = gitlab.import_contributor_from_author(payload)
assert contributor.name == payload["name"]
assert contributor.username == payload["username"]
assert contributor.metadata == {"gitlab": payload}
def test_import_contributor_update_metadata(factories):
existing = factories.Contributor(username="Alice42", metadata={"hello": "world"})
payload = {"id": 130, "name": "Alice", "username": "Alice42"}
contributor = gitlab.import_contributor_from_author(payload)
contributor == existing
assert contributor.name == existing.name
assert contributor.username == existing.username
assert contributor.metadata == {"gitlab": payload, "hello": "world"}
def test_retrieve_issue_page(requests_mock):
gitlab_url = "https://gitlab.test"
project_id = 1
url = f"{gitlab_url}/api/v4/projects/{project_id}/issues"
expected_next_page = (
f"https://gitlab.test/api/v4/projects/{project_id}/issues?page=2"
)
requests_mock.get(
f"https://gitlab.test/api/v4/projects/{project_id}/issues",
json=["hello", "world"],
headers={"Link": f'<{expected_next_page}>; rel="next"'},
)
result, next_page = gitlab.retrieve_issue_page(url)
assert result == ["hello", "world"]
assert expected_next_page == next_page
from django.utils import timezone
from . import factories
from contributions.core import models as core_models
def test_contributor_factory():
c = factories.Contributor(
name='alice',
def test_contributor_factory(factories):
contributor = factories.Contributor(
name="alice", is_visible=True, metadata={"hello": "world"}
)
assert contributor.name == "alice"
assert contributor.creation_date < timezone.now()
assert contributor.is_visible is True
assert contributor.metadata == {"hello": "world"}
def test_contribution_factory(factories):
contribution = factories.Contribution(
external_id="weblate:front",
metadata={"locale": "en"},
is_visible=True,
links={
'hello': 'world'
}
url="https://weblate.test",
summary="Hello",
type="i18n:translation",
)
assert c.name == 'alice'
assert c.creation_date < timezone.now()
assert c.is_visible is True
assert c.links == {
'hello': 'world'
}
assert isinstance(contribution.contributor, core_models.Contributor)
assert contribution.creation_date < timezone.now()
assert contribution.import_date < timezone.now()
assert contribution.is_visible is True
assert contribution.metadata == {"locale": "en"}
assert contribution.summary == "Hello"
assert contribution.type == "i18n:translation"
assert contribution.external_id == "weblate:front"
assert contribution.url == "https://weblate.test"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment