Add a massimport page

diff --git a/quotes/conftest.py b/quotes/conftest.py
index 9e88dc0a43b2797e6618f1ea49d214defc4d454c..5cdf30a8431f17093e24322ca3e8ba8892103adb 100644 (file)
--- a/quotes/conftest.py
+++ b/quotes/conftest.py
@@ -7,10 +7,13 @@ class ValidatingClient(object):
         self.client = client
     
     def request(self, url, method, exp_status=200, params={}):
         self.client = client
     
     def request(self, url, method, exp_status=200, params={}):

+        if not url.startswith('/quotes/'):
+            url = '/quotes/' + url
+

         if method == 'get':
         if method == 'get':

-            response = self.client.get('/quotes/' + url)
+            response = self.client.get(url)

         elif method == 'post':
         elif method == 'post':

-            response = self.client.post('/quotes/' + url, params)
+            response = self.client.post(url, params)

         else:
             raise RuntimeError('Unknown method %s for %s' % (method, url))
         assert response.status_code == exp_status
         else:
             raise RuntimeError('Unknown method %s for %s' % (method, url))
         assert response.status_code == exp_status

diff --git a/quotes/massimport.py b/quotes/massimport.py
new file mode 100644 (file)
index 0000000..ade6f38
--- /dev/null
+++ b/quotes/massimport.py
@@ -0,0 +1,95 @@
+import re
+from django.db import DatabaseError, transaction
+
+from quotes.models import QuoteTag, Quote, Work, Author
+
+def get_or_create_author(authorname, resultcontext):
+    authorname = authorname.strip()
+    try:
+        return Author.objects.get(name=authorname)
+    except Author.DoesNotExist:
+        a = Author.objects.create(name=authorname,
+                                  pvt_notes="<p>Created during mass import</p>")
+        resultcontext["created_authors"] += [a]
+        return a
+
+def get_or_create_work(workname, author, resultcontext):
+    workname = workname.strip()
+    try:
+        return Work.objects.get(name=workname, author=author)
+    except Work.DoesNotExist:
+        w = Work.objects.create(name=workname,
+                                author=author,
+                                pvt_notes="<p>Created during mass import</p>")
+        resultcontext["created_works"] += [w]
+        return w
+
+def get_or_create_tag(tagname, resultcontext):
+    try:
+        return QuoteTag.objects.get(tag=tagname)
+    except QuoteTag.DoesNotExist:
+        t = QuoteTag.objects.create(tag=tagname)
+        resultcontext["created_tags"] += [t]
+        return t
+    
+def add_tags_on_quote(quote, tagline, resultcontext):
+    for tagname in tagline.split(","):
+        tagname = tagname.strip()
+        if not tagname:
+            continue
+        tag = get_or_create_tag(tagname, resultcontext)
+        quote.tags.add(tag)
+
+def paragraphize(text):
+    paragraph = ""
+    for line in text.splitlines():
+        line = line.strip()
+        if not line: continue
+        paragraph += "<p>%s</p>" % line
+        # rest of the HTML will be bleach.clean()'d
+    return paragraph
+        
+def create_quote(quotetext, authorname, workname, tagline, resultcontext):
+    author = get_or_create_author(authorname, resultcontext)
+    work = get_or_create_work(workname, author, resultcontext)
+    quotetext = paragraphize(quotetext)
+    quote = Quote.objects.create(text=quotetext, work=work)
+    add_tags_on_quote(quote, tagline, resultcontext)
+    resultcontext["created_quotes"] += [quote]
+
+def create_all_quotes(allquotes, resultcontext):
+    quotesep = re.compile(r'^\s*===+.*$', re.MULTILINE)
+    elemsep = re.compile(r'^\s*---+.*$', re.MULTILINE)
+    
+    for fullquote in quotesep.split(allquotes):
+        if not fullquote: continue
+
+        elements = elemsep.split(fullquote)
+        if len(elements) not in [3, 4]:
+            resultcontext["rejected"] += [fullquote]
+            continue
+        quotetext, authorname, workname = elements[0:3]
+        if len(elements) == 4:
+            tagline = elements[3]
+        else:
+            tagline = ''
+        create_quote(quotetext, authorname, workname, tagline,
+                     resultcontext)
+
+def domassimport(allquotes):
+    resultcontext = {}
+    resultcontext["created_quotes"] = []
+    resultcontext["created_tags"] = []
+    resultcontext["created_works"] = []
+    resultcontext["created_authors"] = []
+    resultcontext["rejected"] = []
+    resultcontext["fatal_error"] = False
+
+    try:
+        with transaction.atomic():
+            create_all_quotes(allquotes, resultcontext)
+    except DatabaseError as e:
+        resultcontext["fatal_error"] = True
+        resultcontext["fatal_error_message"] = e.__cause__
+
+    return resultcontext

diff --git a/quotes/static/quotes/style.css b/quotes/static/quotes/style.css
index 0d5423b47bcace28443db046056501e94638a04b..454b851f18b00792f18271be17d6f58db1a2ce81 100644 (file)
--- a/quotes/static/quotes/style.css
+++ b/quotes/static/quotes/style.css
@@ -23,7 +23,7 @@ a:hover {
     text-align: right;
 }
 
     text-align: right;
 }
 

-.quote .author .work_name {
+.work_name {

     font-style: italic;
 }
 
     font-style: italic;
 }
 

diff --git a/quotes/templates/quotes/display.html b/quotes/templates/quotes/display.html
index 1b774a1463dfd71135f5ed6039fb7acc71742212..2db4c572c2a38dccf5c48d079a182d24d6873472 100644 (file)
--- a/quotes/templates/quotes/display.html
+++ b/quotes/templates/quotes/display.html
@@ -25,7 +25,7 @@
       {% if not skip_author_notes %}
         {% include "quotes/author_notes.html" with author=quote.work.author %}
       {% endif %}
       {% if not skip_author_notes %}
         {% include "quotes/author_notes.html" with author=quote.work.author %}
       {% endif %}

-      {% if quote.tags.all %}
+      {% if quote.tags.all.count %}

       <p class="tags">
        Tags:
        {% for tag in quote.tags.all %}
       <p class="tags">
        Tags:
        {% for tag in quote.tags.all %}

diff --git a/quotes/templates/quotes/domassimport.html b/quotes/templates/quotes/domassimport.html
new file mode 100644 (file)
index 0000000..38c8712
--- /dev/null
+++ b/quotes/templates/quotes/domassimport.html
@@ -0,0 +1,58 @@
+{% extends 'quotes/base.html' %}
+
+{% block title %}Searching for quotes{% endblock %}
+
+{% block body %}
+
+{% if fatal_error %}
+  <p>Fatal error during import process. Transaction rolled back.</p>
+
+  <p>The error from the database was:</p>
+
+  <pre>{{ fatal_error_message }}</pre>
+{% endif %}
+
+{% if rejected %}
+  <p>Some quotes were rejected during import:</p>
+  {% for reject in rejected %}
+    <p>The quote made up of:</p>
+    <pre class="rejected">{{ reject }}</pre>
+  {% endfor %}
+{% endif %}
+
+<p>Created authors:</p>
+<ul>
+  {% for author in created_authors %}
+  <li><a href="{{ author.get_absolute_url }}"
+        class="author_name">{{ author.name }}</a></li>
+  {% empty %}
+  <li>No new author</li>
+  {% endfor %}
+</ul>
+
+<p>Created works:</p>
+<ul>
+  {% for work in created_works %}
+  <li><a href="{{ work.get_absolute_url }}"
+        class="work_name">{{ work.name }}</a></li>
+  {% empty %}
+  <li>No new work</li>
+  {% endfor %}
+</ul>
+
+<p>Created tags:</p>
+<ul>
+  {% for tag in created_tags %}
+  <li><a href="{{ tag.get_absolute_url }}"
+        class="tag_link">{{ tag.tag }}</a></li>
+  {% empty %}
+  <li>No new tag</li>
+  {% endfor %}
+</ul>
+
+<p>Created quotes:</p>
+{% for quote in created_quotes %}
+  {% include "quotes/display.html" with quote=quote %}
+{% endfor %}
+
+{% endblock %}

diff --git a/quotes/templates/quotes/massimport.html b/quotes/templates/quotes/massimport.html
new file mode 100644 (file)
index 0000000..c24a11b
--- /dev/null
+++ b/quotes/templates/quotes/massimport.html
@@ -0,0 +1,42 @@
+{% extends 'quotes/base.html' %}
+
+{% block title %}Importing several quotes at once{% endblock %}
+
+{% block body %}
+
+<p>Expected format in the text area:</p>
+
+<pre>
+  Text of the quote
+  ---
+  Author of the quote
+  ---
+  Title of the work (book or play title, name of the speech, historical event, etc.)
+  ---
+  tag 1, tag 2, separated by commas
+  ===
+  A second quote
+  ---
+  Author of the second quote
+  ---
+  Title of the second work
+  ---
+  some, tags
+  ===
+  A third quote
+  ---
+  Author of the quote
+  ---
+  Title of the work
+  ---
+  etc, etc
+</pre>
+
+<form action="{% url 'quotes:massimport' %}" method="post">
+  {% csrf_token %}
+  <p>Quotes to import: </p>
+  <textarea name="quotes" cols="100" rows="20"></textarea>
+  <p><input type="submit" /></p>
+</form>
+
+{% endblock %}

diff --git a/quotes/test_massimport.py b/quotes/test_massimport.py
new file mode 100644 (file)
index 0000000..0a9fb4f
--- /dev/null
+++ b/quotes/test_massimport.py
@@ -0,0 +1,121 @@
+import pytest
+
+from .models import Author, Work, Quote, QuoteTag
+
+class Test_MassImport():
+    @pytest.fixture(scope='function')
+    def q1(self, db):
+        a1 = Author.objects.create(name="JFK")
+        w1 = Work.objects.create(name="Berlin speech", author=a1)
+        q1 = Quote.objects.create(text="<p>Ich bin...</p>", work=w1)
+        return q1
+
+    @pytest.mark.django_db
+    def test_massimport_1(self, q1, c):
+        allquotes = """\
+Ich bin ein Berliner
+---
+JFK
+---
+Berlin speech
+===
+To be or not to be, that is the question
+---
+William Shakespeare
+---
+Hamlet
+===
+To thine own self, be true
+---
+William Shakespeare
+---
+Hamlet
+---
+tag1, tag2, tag3
+===
+A rose by any other name...
+---
+William Shakespeare
+---
+Romeo and Juliet
+---
+tag1, tag555
+==="""
+
+        results = c.postPage('massimport/', {'quotes': allquotes})
+
+        assert "rejected" not in results
+
+        assert Quote.objects.get(text="<p>To thine own self, be true</p>")
+        assert Quote.objects.get(text__contains="To be or not to be")
+
+        assert Author.objects.get(name="JFK")
+        assert Author.objects.get(name="William Shakespeare")
+        assert Author.objects.all().count() == 2
+
+        hamlet = Work.objects.get(name="Hamlet")
+        assert hamlet
+        assert hamlet.author == Author.objects.get(name="William Shakespeare")
+
+        for quote in Quote.objects.all():
+            assert c.getPage(quote.get_absolute_url())
+
+    @pytest.mark.django_db
+    def test_massimport_2(self, q1, c):
+        allquotes = """\
+<script>somethingevil()</script>
+A rose by any other name...
+---
+William Shakespeare
+---
+Romeo and Juliet
+---
+tag1, tag555
+==="""
+
+        results = c.postPage('massimport/', {'quotes': allquotes})
+        assert "<script>" not in results
+        with pytest.raises(Quote.DoesNotExist):
+            Quote.objects.get(text__contains="<script>")
+
+    @pytest.mark.django_db
+    def test_massimport_3(self, c):
+        """Whitespace and stuff"""
+        allquotes = """\
+A rose by any other name...
+---
+William Shakespeare
+---
+Romeo and Juliet
+---
+tag1, tag555
+===
+
+To be, or not to be, that is the question:
+Whether 'tis Nobler in the mind to suffer
+The Slings and Arrows of outrageous Fortune,
+Or to take Arms against a Sea of troubles,
+And by opposing end them: to die, to sleep
+No more;
+
+---
+   William Shakespeare  
+
+---
+Hamlet  
+
+"""
+
+        results = c.postPage('massimport/', {'quotes': allquotes})
+        assert "rejected" not in results
+        assert Quote.objects.get(text="<p>A rose by any other name...</p>")
+        assert Author.objects.get(name="William Shakespeare")
+        assert Work.objects.get(name="Romeo and Juliet")
+        assert QuoteTag.objects.get(tag="tag1")
+        assert QuoteTag.objects.get(tag="tag555")
+
+        tirade = Quote.objects.get(text__contains="To be, or not to be")
+        assert tirade.work.name == "Hamlet"
+        assert tirade.work.author.name == "William Shakespeare"
+        assert Author.objects.filter(name__contains="Shakespeare").count() == 1
+

diff --git a/quotes/urls.py b/quotes/urls.py
index 0c43cc47259dffeb47f4e0e19dab07063c57bb2c..55577bb5e7b16fa8cdedc6694642155c542dd438 100644 (file)
--- a/quotes/urls.py
+++ b/quotes/urls.py
@@ -15,6 +15,7 @@ urlpatterns = [
 
     url(r'^cloud/$', views.cloud, name="cloud"),
     url(r'^search/$', views.searchpage, name="search"),
 
     url(r'^cloud/$', views.cloud, name="cloud"),
     url(r'^search/$', views.searchpage, name="search"),

+    url(r'^massimport/$', views.massimport, name="massimport"),

 
     url(r'^all/$', views.all, name="all"),
 ]
 
     url(r'^all/$', views.all, name="all"),
 ]

diff --git a/quotes/views.py b/quotes/views.py
index 01cb4e33f5ced7d73efac3edfd0ae303d26b604f..a0969c113699c57724ff5ed1ca2223f9dbc87001 100644 (file)
--- a/quotes/views.py
+++ b/quotes/views.py
@@ -5,6 +5,7 @@ from random import randint
 from quotes.models import Author, Work, Quote, QuoteTag
 import quotes.search as search
 import quotes.tagcloud as tagcloud
 from quotes.models import Author, Work, Quote, QuoteTag
 import quotes.search as search
 import quotes.tagcloud as tagcloud

+from quotes.massimport import domassimport

 
 # create your views here.
 def index(request):
 
 # create your views here.
 def index(request):


@@ -51,3 +52,10 @@ def searchpage(request):
 def cloud(request):
     clouddata = tagcloud.build_cloud()
     return render(request, 'quotes/cloud.html', { 'cloud': clouddata })
 def cloud(request):
     clouddata = tagcloud.build_cloud()
     return render(request, 'quotes/cloud.html', { 'cloud': clouddata })

+
+def massimport(request):
+    if 'quotes' in request.POST:
+        resultcontext = domassimport(request.POST['quotes'])
+        return render(request, 'quotes/domassimport.html', resultcontext)
+    else:
+        return render(request, 'quotes/massimport.html', {})