Mercurial > public > sg101

comparison gpp/forums/latest.py @ 509:248dd8dd67f8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For #237, use Redis as the source of posts for the RSS feeds to hopefully eliminate some slow queries.
author Brian Neal <bgneal@gmail.com>
date Wed, 07 Dec 2011 01:08:54 +0000
parents
children 82b97697312e
comparison
equal deleted inserted replaced
508:6f5fff924877 509:248dd8dd67f8
1 """
2 This module maintains the latest posts datastore. The latest posts are often
3 needed by RSS feeds, "latest posts" template tags, etc. This module listens for
4 the post_content_update signal, then bundles the post up and stores it by forum
5 ID in Redis. We also maintain a combined forums list. This allows quick
6 retrieval of the latest posts and avoids some slow SQL queries.
7
8 """
9 import datetime
10 import time
11
12 from django.dispatch import receiver
13 from django.utils import simplejson
14
15 from forums.signals import post_content_update
16 from forums.models import Forum
17 from core.services import get_redis_connection
18
19
20 # This constant controls how many latest posts per forum we store
21 MAX_POSTS = 50
22
23
24 @receiver(post_content_update, dispatch_uid='forums.latest_posts')
25 def on_post_update(sender, **kwargs):
26 """
27 This function is our signal handler, called when a post has been updated.
28 We only care about newly created posts, and ignore updates.
29
30 We serialize a post to JSON then store in two lists in Redis:
31 1. The list for the post's parent forum
32 2. The combined forum list
33
34 Note that we only store posts from public forums.
35
36 """
37 # ignore non-new posts
38 if not kwargs['created']:
39 return
40
41 # ignore posts from non-public forums
42 public_forums = Forum.objects.public_forum_ids()
43 if sender.topic.forum.id not in public_forums:
44 return
45
46 # serialize post attributes
47 post_content = {
48 'id': sender.id,
49 'title': sender.topic.name,
50 'content': sender.html,
51 'author': sender.user.username,
52 'pubdate': int(time.mktime(sender.creation_date.timetuple())),
53 'forum_name': sender.topic.forum.name,
54 'url': sender.get_absolute_url()
55 }
56
57 s = simplejson.dumps(post_content)
58
59 # store in Redis
60
61 redis = get_redis_connection()
62 pipeline = redis.pipeline()
63
64 key = 'forums:latest:%d' % sender.topic.forum.id
65
66 pipeline.lpush(key, s)
67 pipeline.ltrim(key, 0, MAX_POSTS - 1)
68
69 # store in the combined feed; yes this wastes some memory storing it twice,
70 # but it makes things much easier
71
72 key = 'forums:latest:*'
73
74 pipeline.lpush(key, s)
75 pipeline.ltrim(key, 0, MAX_POSTS - 1)
76
77 pipeline.execute()
78
79
80 def get_latest_posts(num_posts=MAX_POSTS, forum_id=None):
81 """
82 This function retrieves num_posts latest posts for the forum with the given
83 forum_id. If forum_id is None, the posts are retrieved from the combined
84 forums datastore. A list of dictionaries is returned. Each dictionary
85 contains information about a post.
86
87 """
88 key = 'forums:latest:%d' % forum_id if forum_id else 'forums:latest:*'
89
90 num_posts = max(0, min(MAX_POSTS, num_posts))
91
92 if num_posts == 0:
93 return []
94
95 redis = get_redis_connection()
96 raw_posts = redis.lrange(key, 0, num_posts - 1)
97
98 posts = []
99 for raw_post in raw_posts:
100 post = simplejson.loads(raw_post)
101
102 # fix up the pubdate; turn it back into a datetime object
103 post['pubdate'] = datetime.datetime.fromtimestamp(post['pubdate'])
104
105 posts.append(post)
106
107 return posts