Mercurial > public > sg101

annotate gpp/forums/latest.py @ 509:248dd8dd67f8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For #237, use Redis as the source of posts for the RSS feeds to hopefully eliminate some slow queries.
author Brian Neal <bgneal@gmail.com>
date Wed, 07 Dec 2011 01:08:54 +0000
parents
children 82b97697312e
rev   line source
bgneal@509 1 """
bgneal@509 2 This module maintains the latest posts datastore. The latest posts are often
bgneal@509 3 needed by RSS feeds, "latest posts" template tags, etc. This module listens for
bgneal@509 4 the post_content_update signal, then bundles the post up and stores it by forum
bgneal@509 5 ID in Redis. We also maintain a combined forums list. This allows quick
bgneal@509 6 retrieval of the latest posts and avoids some slow SQL queries.
bgneal@509 7
bgneal@509 8 """
bgneal@509 9 import datetime
bgneal@509 10 import time
bgneal@509 11
bgneal@509 12 from django.dispatch import receiver
bgneal@509 13 from django.utils import simplejson
bgneal@509 14
bgneal@509 15 from forums.signals import post_content_update
bgneal@509 16 from forums.models import Forum
bgneal@509 17 from core.services import get_redis_connection
bgneal@509 18
bgneal@509 19
bgneal@509 20 # This constant controls how many latest posts per forum we store
bgneal@509 21 MAX_POSTS = 50
bgneal@509 22
bgneal@509 23
bgneal@509 24 @receiver(post_content_update, dispatch_uid='forums.latest_posts')
bgneal@509 25 def on_post_update(sender, **kwargs):
bgneal@509 26 """
bgneal@509 27 This function is our signal handler, called when a post has been updated.
bgneal@509 28 We only care about newly created posts, and ignore updates.
bgneal@509 29
bgneal@509 30 We serialize a post to JSON then store in two lists in Redis:
bgneal@509 31 1. The list for the post's parent forum
bgneal@509 32 2. The combined forum list
bgneal@509 33
bgneal@509 34 Note that we only store posts from public forums.
bgneal@509 35
bgneal@509 36 """
bgneal@509 37 # ignore non-new posts
bgneal@509 38 if not kwargs['created']:
bgneal@509 39 return
bgneal@509 40
bgneal@509 41 # ignore posts from non-public forums
bgneal@509 42 public_forums = Forum.objects.public_forum_ids()
bgneal@509 43 if sender.topic.forum.id not in public_forums:
bgneal@509 44 return
bgneal@509 45
bgneal@509 46 # serialize post attributes
bgneal@509 47 post_content = {
bgneal@509 48 'id': sender.id,
bgneal@509 49 'title': sender.topic.name,
bgneal@509 50 'content': sender.html,
bgneal@509 51 'author': sender.user.username,
bgneal@509 52 'pubdate': int(time.mktime(sender.creation_date.timetuple())),
bgneal@509 53 'forum_name': sender.topic.forum.name,
bgneal@509 54 'url': sender.get_absolute_url()
bgneal@509 55 }
bgneal@509 56
bgneal@509 57 s = simplejson.dumps(post_content)
bgneal@509 58
bgneal@509 59 # store in Redis
bgneal@509 60
bgneal@509 61 redis = get_redis_connection()
bgneal@509 62 pipeline = redis.pipeline()
bgneal@509 63
bgneal@509 64 key = 'forums:latest:%d' % sender.topic.forum.id
bgneal@509 65
bgneal@509 66 pipeline.lpush(key, s)
bgneal@509 67 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 68
bgneal@509 69 # store in the combined feed; yes this wastes some memory storing it twice,
bgneal@509 70 # but it makes things much easier
bgneal@509 71
bgneal@509 72 key = 'forums:latest:*'
bgneal@509 73
bgneal@509 74 pipeline.lpush(key, s)
bgneal@509 75 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 76
bgneal@509 77 pipeline.execute()
bgneal@509 78
bgneal@509 79
bgneal@509 80 def get_latest_posts(num_posts=MAX_POSTS, forum_id=None):
bgneal@509 81 """
bgneal@509 82 This function retrieves num_posts latest posts for the forum with the given
bgneal@509 83 forum_id. If forum_id is None, the posts are retrieved from the combined
bgneal@509 84 forums datastore. A list of dictionaries is returned. Each dictionary
bgneal@509 85 contains information about a post.
bgneal@509 86
bgneal@509 87 """
bgneal@509 88 key = 'forums:latest:%d' % forum_id if forum_id else 'forums:latest:*'
bgneal@509 89
bgneal@509 90 num_posts = max(0, min(MAX_POSTS, num_posts))
bgneal@509 91
bgneal@509 92 if num_posts == 0:
bgneal@509 93 return []
bgneal@509 94
bgneal@509 95 redis = get_redis_connection()
bgneal@509 96 raw_posts = redis.lrange(key, 0, num_posts - 1)
bgneal@509 97
bgneal@509 98 posts = []
bgneal@509 99 for raw_post in raw_posts:
bgneal@509 100 post = simplejson.loads(raw_post)
bgneal@509 101
bgneal@509 102 # fix up the pubdate; turn it back into a datetime object
bgneal@509 103 post['pubdate'] = datetime.datetime.fromtimestamp(post['pubdate'])
bgneal@509 104
bgneal@509 105 posts.append(post)
bgneal@509 106
bgneal@509 107 return posts