Mercurial > public > sg101

changeset 399:24f1230f3ee3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixing #193; reduce news query counts by grabbing tags and comment counts in bulk. Increased news items per page from 5 to 10.
author Brian Neal <bgneal@gmail.com>
date Sat, 26 Mar 2011 03:08:05 +0000
parents 701730b2fcda
children 47f4259ce511
files gpp/comments/models.py gpp/news/templatetags/news_tags.py gpp/news/utils.py gpp/news/views.py gpp/templates/news/story_summary.html
diffstat 5 files changed, 197 insertions(+), 133 deletions(-) [+]
line wrap: on
line diff
--- a/gpp/comments/models.py	Sat Mar 26 00:26:00 2011 +0000
+++ b/gpp/comments/models.py	Sat Mar 26 03:08:05 2011 +0000
@@ -31,7 +31,7 @@
 class Comment(models.Model):
     """My own version of a Comment class that can attach comments to any other model."""
     content_type = models.ForeignKey(ContentType)
-    object_id = models.PositiveIntegerField()
+    object_id = models.PositiveIntegerField(db_index=True)
     content_object = generic.GenericForeignKey('content_type', 'object_id')
     user = models.ForeignKey(User)
     comment = models.TextField(max_length=COMMENT_MAX_LENGTH)
--- a/gpp/news/templatetags/news_tags.py	Sat Mar 26 00:26:00 2011 +0000
+++ b/gpp/news/templatetags/news_tags.py	Sat Mar 26 03:08:05 2011 +0000
@@ -7,6 +7,7 @@
 from django.conf import settings
 
 from news.models import Story
+from news.utils import attach_extra_attrs
 
 
 register = template.Library()
@@ -14,8 +15,13 @@
 
 @register.inclusion_tag('news/current_news.html')
 def current_news():
-    stories = Story.objects.exclude(
+    # Defer the tags field because we are going to get all the
+    # tags out in 1 query later...
+    stories = Story.objects.defer('tags').exclude(
             front_page_expiration__lt=datetime.date.today())[:10]
+
+    attach_extra_attrs(stories)
+
     return {
         'stories': stories,
         'STATIC_URL': settings.STATIC_URL,
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/gpp/news/utils.py	Sat Mar 26 03:08:05 2011 +0000
@@ -0,0 +1,44 @@
+"""
+Common utility/helper code for the news app.
+
+"""
+from django.contrib.contenttypes.models import ContentType
+
+from comments.models import Comment
+from tagging.models import TaggedItem
+from news.models import Story
+
+
+def attach_extra_attrs(stories):
+    """
+    For each story in the input stories list, attach 2 new attributes:
+    tag_list and comment_count. The tags and comment count info is pulled from
+    the database in bulk. This saves database queries when lots of news
+    stories are displayed at once. For best results, use ".defer('tags')"
+    when retrieve the stories from the database.
+
+    """
+    stories_dict = dict((story.id, story) for story in stories)
+    story_ids = stories_dict.keys()
+
+    # Get all the tags out in one query
+    ct = ContentType.objects.get_for_model(Story)
+    tagged_items = TaggedItem.objects.filter(content_type=ct,
+            object_id__in=story_ids).select_related('tag')
+
+    for story in stories_dict.values():
+        story.tag_list = []
+        story.comment_count = 0
+
+    # attach tags
+    for item in tagged_items:
+        stories_dict[item.object_id].tag_list.append(item.tag.name)
+
+    # Now get all the comment counts out in one fell swoop
+
+    story_ids = Comment.objects.filter(content_type=ct,
+            object_id__in=story_ids).values_list('object_id', flat=True)
+
+    # compute comment_count
+    for story_id in story_ids:
+        stories_dict[story_id].comment_count += 1
--- a/gpp/news/views.py	Sat Mar 26 00:26:00 2011 +0000
+++ b/gpp/news/views.py	Sat Mar 26 03:08:05 2011 +0000
@@ -28,198 +28,214 @@
 from news.models import Story
 from news.forms import AddNewsForm
 from news.forms import SendStoryForm
+from news.utils import attach_extra_attrs
 
-NEWS_PER_PAGE = 5
+
+NEWS_PER_PAGE = 10
 
 #######################################################################
 
 def create_paginator(stories):
-   return DiggPaginator(stories, NEWS_PER_PAGE, body=5, tail=3, margin=3, padding=2)
+    return DiggPaginator(stories, NEWS_PER_PAGE, body=5, tail=3, margin=3, padding=2)
 
 #######################################################################
 
 def index(request):
-   stories = Story.objects.all().select_related()
-   paginator = create_paginator(stories)
+    # Defer the tags field because we are going to get all the
+    # tags out in 1 query later...
+    stories = Story.objects.all().defer('tags').select_related()
+    paginator = create_paginator(stories)
 
-   page = get_page(request.GET)
-   try:
-      the_page = paginator.page(page)
-   except InvalidPage:
-      raise Http404
+    page = get_page(request.GET)
+    try:
+        the_page = paginator.page(page)
+    except InvalidPage:
+        raise Http404
 
-   return render_to_response('news/index.html', {
-      'title': 'Main Index',
-      'page': the_page,
-      }, 
-      context_instance = RequestContext(request))
+    # Go get the tags and comment counts for all these stories in bulk rather
+    # than one at a time in the template; this saves database queries
+    attach_extra_attrs(the_page.object_list)
+
+    return render_to_response('news/index.html', {
+        'title': 'Main Index',
+        'page': the_page,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def archive_index(request):
-   dates = Story.objects.dates('date_submitted', 'month', order='DESC')
-   return render_to_response('news/archive_index.html', {
-      'title': 'News Archive',
-      'dates': dates, 
-      }, 
-      context_instance = RequestContext(request))
+    dates = Story.objects.dates('date_submitted', 'month', order='DESC')
+    return render_to_response('news/archive_index.html', {
+        'title': 'News Archive',
+        'dates': dates,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def archive(request, year, month):
-   stories = Story.objects.filter(date_submitted__year=year, date_submitted__month=month)
-   paginator = create_paginator(stories)
-   page = get_page(request.GET)
-   try:
-      the_page = paginator.page(page)
-   except InvalidPage:
-      raise Http404
+    stories = Story.objects.defer('tags').filter(date_submitted__year=year,
+            date_submitted__month=month).select_related()
+    paginator = create_paginator(stories)
+    page = get_page(request.GET)
+    try:
+        the_page = paginator.page(page)
+    except InvalidPage:
+        raise Http404
 
-   month_name = datetime.date(int(year), int(month), 1).strftime('%B')
+    attach_extra_attrs(the_page.object_list)
 
-   return render_to_response('news/index.html', {
-      'title': 'Archive for %s, %s' % (month_name, year),
-      'page': the_page,
-      }, 
-      context_instance = RequestContext(request))
+    month_name = datetime.date(int(year), int(month), 1).strftime('%B')
+
+    return render_to_response('news/index.html', {
+        'title': 'Archive for %s, %s' % (month_name, year),
+        'page': the_page,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def category_index(request):
-   categories = Category.objects.all().select_related()
-   cat_list = []
-   for cat in categories:
-      cat_list.append((cat, cat.story_set.defer('tags')[:10]))
+    categories = Category.objects.all().select_related()
+    cat_list = []
+    for cat in categories:
+        cat_list.append((cat, cat.story_set.defer('tags')[:10]))
 
-   return render_to_response('news/category_index.html', {
-      'cat_list': cat_list, 
-      }, 
-      context_instance = RequestContext(request))
+    return render_to_response('news/category_index.html', {
+        'cat_list': cat_list,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def category(request, slug):
-   category = get_object_or_404(Category, slug=slug)
-   stories = Story.objects.filter(category=category)
-   paginator = create_paginator(stories)
-   page = get_page(request.GET)
-   try:
-      the_page = paginator.page(page)
-   except InvalidPage:
-      raise Http404
+    category = get_object_or_404(Category, slug=slug)
+    stories = Story.objects.defer('tags').filter(category=category).select_related()
+    paginator = create_paginator(stories)
+    page = get_page(request.GET)
+    try:
+        the_page = paginator.page(page)
+    except InvalidPage:
+        raise Http404
 
-   return render_to_response('news/index.html', {
-      'title': 'Category: ' + category.title,
-      'page': the_page,
-      }, 
-      context_instance = RequestContext(request))
+    attach_extra_attrs(the_page.object_list)
+
+    return render_to_response('news/index.html', {
+        'title': 'Category: ' + category.title,
+        'page': the_page,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def story(request, story_id):
-   story = get_object_or_404(Story, pk=story_id)
-   return render_to_response('news/story.html', {
-      'story': story,
-      },
-      context_instance=RequestContext(request))
+    story = get_object_or_404(Story, pk=story_id)
+    return render_to_response('news/story.html', {
+        'story': story,
+        },
+        context_instance=RequestContext(request))
 
 #######################################################################
 
 @login_required
 def submit(request):
-   if request.method == "POST":
-      add_form = AddNewsForm(request.POST)
-      if add_form.is_valid():
-         pending_story = add_form.save(commit=False)
-         pending_story.submitter = request.user
-         pending_story.short_text = clean_html(pending_story.short_text)
-         pending_story.long_text = clean_html(pending_story.long_text)
-         pending_story.save()
-         return HttpResponseRedirect(reverse('news.views.submit_thanks'))
-   else:
-      add_form = AddNewsForm()
+    if request.method == "POST":
+        add_form = AddNewsForm(request.POST)
+        if add_form.is_valid():
+            pending_story = add_form.save(commit=False)
+            pending_story.submitter = request.user
+            pending_story.short_text = clean_html(pending_story.short_text)
+            pending_story.long_text = clean_html(pending_story.long_text)
+            pending_story.save()
+            return HttpResponseRedirect(reverse('news.views.submit_thanks'))
+    else:
+        add_form = AddNewsForm()
 
-   return render_to_response('news/submit_news.html', {
-      'add_form': add_form,
-      },
-      context_instance = RequestContext(request))
+    return render_to_response('news/submit_news.html', {
+        'add_form': add_form,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 @login_required
 def submit_thanks(request):
-   return render_to_response('news/submit_news.html', {
-      },
-      context_instance = RequestContext(request))
+    return render_to_response('news/submit_news.html', {
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def tags(request):
-   tags = Tag.objects.cloud_for_model(Story)
-   return render_to_response('news/tag_index.html', {
-      'tags': tags,
-      },
-      context_instance = RequestContext(request))
+    tags = Tag.objects.cloud_for_model(Story)
+    return render_to_response('news/tag_index.html', {
+        'tags': tags,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 def tag(request, tag_name):
-   tag = get_object_or_404(Tag, name=tag_name)
-   stories = TaggedItem.objects.get_by_model(Story.objects.all().select_related(), tag)
-   paginator = create_paginator(stories)
-   page = get_page(request.GET)
-   try:
-      the_page = paginator.page(page)
-   except InvalidPage:
-      raise Http404
+    tag = get_object_or_404(Tag, name=tag_name)
+    stories = TaggedItem.objects.get_by_model(
+            Story.objects.defer('tags').select_related(), tag)
+    paginator = create_paginator(stories)
+    page = get_page(request.GET)
+    try:
+        the_page = paginator.page(page)
+    except InvalidPage:
+        raise Http404
 
-   return render_to_response('news/index.html', {
-      'title': 'Stories with tag: "%s"' % tag_name,
-      'page': the_page,
-      }, 
-      context_instance=RequestContext(request))
+    attach_extra_attrs(the_page.object_list)
+
+    return render_to_response('news/index.html', {
+        'title': 'Stories with tag: "%s"' % tag_name,
+        'page': the_page,
+        },
+        context_instance=RequestContext(request))
 
 #######################################################################
 
 @login_required
 def email_story(request, story_id):
-   story = get_object_or_404(Story, pk=story_id)
-   if request.method == 'POST':
-      send_form = SendStoryForm(request.POST)
-      if send_form.is_valid():
-         to_name = send_form.name()
-         to_email = send_form.email()
-         from_name = get_full_name(request.user)
-         from_email = request.user.email
-         site = Site.objects.get_current()
+    story = get_object_or_404(Story, pk=story_id)
+    if request.method == 'POST':
+        send_form = SendStoryForm(request.POST)
+        if send_form.is_valid():
+            to_name = send_form.name()
+            to_email = send_form.email()
+            from_name = get_full_name(request.user)
+            from_email = request.user.email
+            site = Site.objects.get_current()
 
-         msg = render_to_string('news/send_story_email.txt',
-               {
-                  'to_name': to_name,
-                  'sender_name': from_name,
-                  'site_name' : site.name,
-                  'site_url' : site.domain,
-                  'story_title': story.title,
-                  'story_link': story.get_absolute_url(),
-               })
+            msg = render_to_string('news/send_story_email.txt',
+                    {
+                        'to_name': to_name,
+                        'sender_name': from_name,
+                        'site_name' : site.name,
+                        'site_url' : site.domain,
+                        'story_title': story.title,
+                        'story_link': story.get_absolute_url(),
+                    })
 
-         subject = 'Interesting Story at ' + site.name
-         send_mail(subject, msg, from_email, [to_email])
-         return HttpResponseRedirect(reverse('news.views.email_thanks'))
-   else:
-      send_form = SendStoryForm()
+            subject = 'Interesting Story at ' + site.name
+            send_mail(subject, msg, from_email, [to_email])
+            return HttpResponseRedirect(reverse('news.views.email_thanks'))
+    else:
+        send_form = SendStoryForm()
 
-   return render_to_response('news/send_story.html', {
-      'send_form': send_form,
-      'story': story,
-      },
-      context_instance = RequestContext(request))
+    return render_to_response('news/send_story.html', {
+        'send_form': send_form,
+        'story': story,
+        },
+        context_instance = RequestContext(request))
 
 #######################################################################
 
 @login_required
 def email_thanks(request):
-   return render_to_response('news/send_story.html', {
-      },
-      context_instance = RequestContext(request))
+    return render_to_response('news/send_story.html', {
+        },
+        context_instance = RequestContext(request))
 
--- a/gpp/templates/news/story_summary.html	Sat Mar 26 00:26:00 2011 +0000
+++ b/gpp/templates/news/story_summary.html	Sat Mar 26 03:08:05 2011 +0000
@@ -1,7 +1,6 @@
 {% load url from future %}
 {% load tagging_tags %}
 {% load comment_tags %}
-{% get_comment_count for story as comment_count %}
 <div class="news-story-container solid-background">
 {% if on_home %}
 <h3><a href="{{ story.get_absolute_url }}">{{ story.title }}</a></h3>
@@ -27,19 +26,18 @@
 <p>
 Category: <a href="{% url 'news-category' slug=story.category.slug %}">{{ story.category.title }}</a>
 <img src="{{ STATIC_URL }}icons/comments.png" alt="Comments" title="Comments" />
-<a href="{{ story.get_absolute_url }}">{{ comment_count }} comment{{ comment_count|pluralize }}</a>
+<a href="{{ story.get_absolute_url }}">{{ story.comment_count }} comment{{ story.comment_count|pluralize }}</a>
 <a href="{{ story.get_absolute_url }}"><img src="{{ STATIC_URL }}icons/link.png" alt="Permalink" title="Permalink" /></a>
 {% if user.is_authenticated %}
 <a href="{% url 'news.views.email_story' story.id %}"><img src="{{ STATIC_URL }}icons/email_go.png"
    alt="Send this story to a friend" title="Send this story to a friend" /></a>
 {% endif %}
 </p>
-{% tags_for_object story as story_tags %}
-{% if story_tags %}
+{% if story.tag_list %}
 <div class="news-tags">
    <img src="{{ STATIC_URL }}icons/tag_blue.png" alt="Tags" title="Tags" /> Tags:
    <ul>
-      {% for tag in story_tags %}
+      {% for tag in story.tag_list %}
          <li><a href="{% url 'news-tag_page' tag_name=tag %}">{{ tag }}</a></li>
       {% endfor %}
    </ul>