From bb8b5bd36c5f921ba31635bb858db3d4788ef0e9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fr=C3=A9d=C3=A9ric=20Perrin?= Date: Sun, 6 Nov 2016 09:45:55 +0000 Subject: [PATCH] Add a massimport page --- quotes/conftest.py | 7 +- quotes/massimport.py | 95 +++++++++++++++++ quotes/static/quotes/style.css | 2 +- quotes/templates/quotes/display.html | 2 +- quotes/templates/quotes/domassimport.html | 58 +++++++++++ quotes/templates/quotes/massimport.html | 42 ++++++++ quotes/test_massimport.py | 121 ++++++++++++++++++++++ quotes/urls.py | 1 + quotes/views.py | 8 ++ 9 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 quotes/massimport.py create mode 100644 quotes/templates/quotes/domassimport.html create mode 100644 quotes/templates/quotes/massimport.html create mode 100644 quotes/test_massimport.py diff --git a/quotes/conftest.py b/quotes/conftest.py index 9e88dc0..5cdf30a 100644 --- a/quotes/conftest.py +++ b/quotes/conftest.py @@ -7,10 +7,13 @@ class ValidatingClient(object): self.client = client def request(self, url, method, exp_status=200, params={}): + if not url.startswith('/quotes/'): + url = '/quotes/' + url + if method == 'get': - response = self.client.get('/quotes/' + url) + response = self.client.get(url) elif method == 'post': - response = self.client.post('/quotes/' + url, params) + response = self.client.post(url, params) else: raise RuntimeError('Unknown method %s for %s' % (method, url)) assert response.status_code == exp_status diff --git a/quotes/massimport.py b/quotes/massimport.py new file mode 100644 index 0000000..ade6f38 --- /dev/null +++ b/quotes/massimport.py @@ -0,0 +1,95 @@ +import re +from django.db import DatabaseError, transaction + +from quotes.models import QuoteTag, Quote, Work, Author + +def get_or_create_author(authorname, resultcontext): + authorname = authorname.strip() + try: + return Author.objects.get(name=authorname) + except Author.DoesNotExist: + a = Author.objects.create(name=authorname, + pvt_notes="

Created during mass import

") + resultcontext["created_authors"] += [a] + return a + +def get_or_create_work(workname, author, resultcontext): + workname = workname.strip() + try: + return Work.objects.get(name=workname, author=author) + except Work.DoesNotExist: + w = Work.objects.create(name=workname, + author=author, + pvt_notes="

Created during mass import

") + resultcontext["created_works"] += [w] + return w + +def get_or_create_tag(tagname, resultcontext): + try: + return QuoteTag.objects.get(tag=tagname) + except QuoteTag.DoesNotExist: + t = QuoteTag.objects.create(tag=tagname) + resultcontext["created_tags"] += [t] + return t + +def add_tags_on_quote(quote, tagline, resultcontext): + for tagname in tagline.split(","): + tagname = tagname.strip() + if not tagname: + continue + tag = get_or_create_tag(tagname, resultcontext) + quote.tags.add(tag) + +def paragraphize(text): + paragraph = "" + for line in text.splitlines(): + line = line.strip() + if not line: continue + paragraph += "

%s

" % line + # rest of the HTML will be bleach.clean()'d + return paragraph + +def create_quote(quotetext, authorname, workname, tagline, resultcontext): + author = get_or_create_author(authorname, resultcontext) + work = get_or_create_work(workname, author, resultcontext) + quotetext = paragraphize(quotetext) + quote = Quote.objects.create(text=quotetext, work=work) + add_tags_on_quote(quote, tagline, resultcontext) + resultcontext["created_quotes"] += [quote] + +def create_all_quotes(allquotes, resultcontext): + quotesep = re.compile(r'^\s*===+.*$', re.MULTILINE) + elemsep = re.compile(r'^\s*---+.*$', re.MULTILINE) + + for fullquote in quotesep.split(allquotes): + if not fullquote: continue + + elements = elemsep.split(fullquote) + if len(elements) not in [3, 4]: + resultcontext["rejected"] += [fullquote] + continue + quotetext, authorname, workname = elements[0:3] + if len(elements) == 4: + tagline = elements[3] + else: + tagline = '' + create_quote(quotetext, authorname, workname, tagline, + resultcontext) + +def domassimport(allquotes): + resultcontext = {} + resultcontext["created_quotes"] = [] + resultcontext["created_tags"] = [] + resultcontext["created_works"] = [] + resultcontext["created_authors"] = [] + resultcontext["rejected"] = [] + resultcontext["fatal_error"] = False + + try: + with transaction.atomic(): + create_all_quotes(allquotes, resultcontext) + except DatabaseError as e: + resultcontext["fatal_error"] = True + resultcontext["fatal_error_message"] = e.__cause__ + + return resultcontext diff --git a/quotes/static/quotes/style.css b/quotes/static/quotes/style.css index 0d5423b..454b851 100644 --- a/quotes/static/quotes/style.css +++ b/quotes/static/quotes/style.css @@ -23,7 +23,7 @@ a:hover { text-align: right; } -.quote .author .work_name { +.work_name { font-style: italic; } diff --git a/quotes/templates/quotes/display.html b/quotes/templates/quotes/display.html index 1b774a1..2db4c57 100644 --- a/quotes/templates/quotes/display.html +++ b/quotes/templates/quotes/display.html @@ -25,7 +25,7 @@ {% if not skip_author_notes %} {% include "quotes/author_notes.html" with author=quote.work.author %} {% endif %} - {% if quote.tags.all %} + {% if quote.tags.all.count %}

Tags: {% for tag in quote.tags.all %} diff --git a/quotes/templates/quotes/domassimport.html b/quotes/templates/quotes/domassimport.html new file mode 100644 index 0000000..38c8712 --- /dev/null +++ b/quotes/templates/quotes/domassimport.html @@ -0,0 +1,58 @@ +{% extends 'quotes/base.html' %} + +{% block title %}Searching for quotes{% endblock %} + +{% block body %} + +{% if fatal_error %} +

Fatal error during import process. Transaction rolled back.

+ +

The error from the database was:

+ + 
{{ fatal_error_message }}
+{% endif %} + +{% if rejected %} +

Some quotes were rejected during import:

+ {% for reject in rejected %} +

The quote made up of:

+ 
{{ reject }}
+ {% endfor %} +{% endif %} + +

Created authors:

+ + +

Created works:

+ + +

Created tags:

+ + +

Created quotes:

+{% for quote in created_quotes %} + {% include "quotes/display.html" with quote=quote %} +{% endfor %} + +{% endblock %} diff --git a/quotes/templates/quotes/massimport.html b/quotes/templates/quotes/massimport.html new file mode 100644 index 0000000..c24a11b --- /dev/null +++ b/quotes/templates/quotes/massimport.html @@ -0,0 +1,42 @@ +{% extends 'quotes/base.html' %} + +{% block title %}Importing several quotes at once{% endblock %} + +{% block body %} + +

Expected format in the text area:

+ +
+  Text of the quote
+  ---
+  Author of the quote
+  ---
+  Title of the work (book or play title, name of the speech, historical event, etc.)
+  ---
+  tag 1, tag 2, separated by commas
+  ===
+  A second quote
+  ---
+  Author of the second quote
+  ---
+  Title of the second work
+  ---
+  some, tags
+  ===
+  A third quote
+  ---
+  Author of the quote
+  ---
+  Title of the work
+  ---
+  etc, etc
+
+ +
+ {% csrf_token %} +

Quotes to import:

+ +

+
+ +{% endblock %} diff --git a/quotes/test_massimport.py b/quotes/test_massimport.py new file mode 100644 index 0000000..0a9fb4f --- /dev/null +++ b/quotes/test_massimport.py @@ -0,0 +1,121 @@ +import pytest + +from .models import Author, Work, Quote, QuoteTag + +class Test_MassImport(): + @pytest.fixture(scope='function') + def q1(self, db): + a1 = Author.objects.create(name="JFK") + w1 = Work.objects.create(name="Berlin speech", author=a1) + q1 = Quote.objects.create(text="

Ich bin...

", work=w1) + return q1 + + @pytest.mark.django_db + def test_massimport_1(self, q1, c): + allquotes = """\ +Ich bin ein Berliner +--- +JFK +--- +Berlin speech +=== +To be or not to be, that is the question +--- +William Shakespeare +--- +Hamlet +=== +To thine own self, be true +--- +William Shakespeare +--- +Hamlet +--- +tag1, tag2, tag3 +=== +A rose by any other name... +--- +William Shakespeare +--- +Romeo and Juliet +--- +tag1, tag555 +===""" + + results = c.postPage('massimport/', {'quotes': allquotes}) + + assert "rejected" not in results + + assert Quote.objects.get(text="

To thine own self, be true

") + assert Quote.objects.get(text__contains="To be or not to be") + + assert Author.objects.get(name="JFK") + assert Author.objects.get(name="William Shakespeare") + assert Author.objects.all().count() == 2 + + hamlet = Work.objects.get(name="Hamlet") + assert hamlet + assert hamlet.author == Author.objects.get(name="William Shakespeare") + + for quote in Quote.objects.all(): + assert c.getPage(quote.get_absolute_url()) + + @pytest.mark.django_db + def test_massimport_2(self, q1, c): + allquotes = """\ + +A rose by any other name... +--- +William Shakespeare +--- +Romeo and Juliet +--- +tag1, tag555 +===""" + + results = c.postPage('massimport/', {'quotes': allquotes}) + assert "