Mercurial > public > sg101

annotate gpp/forums/latest.py @ 522:82b97697312e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Created Celery tasks to process new posts and topics. Keep the updated topic set in Redis. This is for tickets #194, #237, #239.
author Brian Neal <bgneal@gmail.com>
date Sun, 18 Dec 2011 23:46:52 +0000
parents 248dd8dd67f8
children e9c446a64423
rev   line source
bgneal@509 1 """
bgneal@509 2 This module maintains the latest posts datastore. The latest posts are often
bgneal@509 3 needed by RSS feeds, "latest posts" template tags, etc. This module listens for
bgneal@509 4 the post_content_update signal, then bundles the post up and stores it by forum
bgneal@509 5 ID in Redis. We also maintain a combined forums list. This allows quick
bgneal@509 6 retrieval of the latest posts and avoids some slow SQL queries.
bgneal@509 7
bgneal@522 8 We also do things like send topic notification emails, auto-favorite, and
bgneal@522 9 auto-subscribe functions here rather than bog the user down in the request /
bgneal@522 10 response cycle.
bgneal@522 11
bgneal@509 12 """
bgneal@509 13 import datetime
bgneal@522 14 import logging
bgneal@509 15 import time
bgneal@509 16
bgneal@509 17 from django.dispatch import receiver
bgneal@509 18 from django.utils import simplejson
bgneal@509 19
bgneal@522 20 from forums.signals import post_content_update, topic_content_update
bgneal@522 21 from forums.models import Forum, Topic, Post
bgneal@522 22 from forums.views.subscriptions import notify_topic_subscribers
bgneal@522 23 from forums.tools import auto_favorite, auto_subscribe
bgneal@509 24 from core.services import get_redis_connection
bgneal@509 25
bgneal@509 26 # This constant controls how many latest posts per forum we store
bgneal@509 27 MAX_POSTS = 50
bgneal@509 28
bgneal@522 29 # This controls how many updated topics we track
bgneal@522 30 MAX_UPDATED_TOPICS = 50
bgneal@522 31
bgneal@522 32 # Redis key names:
bgneal@522 33 POST_COUNT_KEY = "forums:public_post_count"
bgneal@522 34 TOPIC_COUNT_KEY = "forums:public_topic_count"
bgneal@522 35 UPDATED_TOPICS_SET_KEY = "forums:updated_topics:set"
bgneal@522 36 UPDATED_TOPIC_KEY = "forums:updated_topics:%s"
bgneal@522 37
bgneal@522 38 logger = logging.getLogger(__name__)
bgneal@522 39
bgneal@509 40
bgneal@509 41 @receiver(post_content_update, dispatch_uid='forums.latest_posts')
bgneal@509 42 def on_post_update(sender, **kwargs):
bgneal@509 43 """
bgneal@509 44 This function is our signal handler, called when a post has been updated.
bgneal@509 45 We only care about newly created posts, and ignore updates.
bgneal@509 46
bgneal@522 47 We kick off a Celery task to perform work outside of the request/response
bgneal@522 48 cycle.
bgneal@509 49
bgneal@509 50 """
bgneal@509 51 # ignore non-new posts
bgneal@509 52 if not kwargs['created']:
bgneal@509 53 return
bgneal@509 54
bgneal@522 55 # Kick off a Celery task to process this new post
bgneal@522 56 forums.tasks.new_post_task.delay(sender.id)
bgneal@522 57
bgneal@522 58
bgneal@522 59 def process_new_post(post_id):
bgneal@522 60 """
bgneal@522 61 This function is run on a Celery task. It performs all new-post processing.
bgneal@522 62
bgneal@522 63 """
bgneal@522 64 try:
bgneal@522 65 post = Post.objects.select_related().get(pk=post_id)
bgneal@522 66 except Post.DoesNotExist:
bgneal@522 67 logger.warning("process_new_post: post %d does not exist", post_id)
bgneal@509 68 return
bgneal@509 69
bgneal@522 70 # selectively process posts from non-public forums
bgneal@522 71 public_forums = Forum.objects.public_forum_ids()
bgneal@522 72
bgneal@522 73 if post.topic.forum.id in public_forums:
bgneal@522 74 redis = get_redis_connection()
bgneal@522 75 _update_post_feeds(redis, post)
bgneal@522 76 _update_post_count(redis, public_forums)
bgneal@522 77 _update_latest_topics(redis, post)
bgneal@522 78
bgneal@522 79 # send out any email notifications
bgneal@522 80 notify_topic_subscribers(post, defer=False)
bgneal@522 81
bgneal@522 82 # perform any auto-favorite and auto-subscribe actions for the new post
bgneal@522 83 auto_favorite(post)
bgneal@522 84 auto_subscribe(post)
bgneal@522 85
bgneal@522 86
bgneal@522 87 def _update_post_feeds(redis, post):
bgneal@522 88 """
bgneal@522 89 Updates the forum feeds we keep in Redis so that our RSS feeds are quick.
bgneal@522 90
bgneal@522 91 """
bgneal@509 92 # serialize post attributes
bgneal@509 93 post_content = {
bgneal@522 94 'id': post.id,
bgneal@522 95 'title': post.topic.name,
bgneal@522 96 'content': post.html,
bgneal@522 97 'author': post.user.username,
bgneal@522 98 'pubdate': int(time.mktime(post.creation_date.timetuple())),
bgneal@522 99 'forum_name': post.topic.forum.name,
bgneal@522 100 'url': post.get_absolute_url()
bgneal@509 101 }
bgneal@509 102
bgneal@509 103 s = simplejson.dumps(post_content)
bgneal@509 104
bgneal@509 105 # store in Redis
bgneal@509 106
bgneal@509 107 pipeline = redis.pipeline()
bgneal@509 108
bgneal@522 109 key = 'forums:latest:%d' % post.topic.forum.id
bgneal@509 110
bgneal@509 111 pipeline.lpush(key, s)
bgneal@509 112 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 113
bgneal@509 114 # store in the combined feed; yes this wastes some memory storing it twice,
bgneal@509 115 # but it makes things much easier
bgneal@509 116
bgneal@509 117 key = 'forums:latest:*'
bgneal@509 118
bgneal@509 119 pipeline.lpush(key, s)
bgneal@509 120 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 121
bgneal@509 122 pipeline.execute()
bgneal@509 123
bgneal@509 124
bgneal@522 125 def _update_post_count(redis, public_forums):
bgneal@522 126 """
bgneal@522 127 Updates the post count we cache in Redis. Doing a COUNT(*) on the post table
bgneal@522 128 can be expensive in MySQL InnoDB.
bgneal@522 129
bgneal@522 130 """
bgneal@522 131 result = redis.incr(POST_COUNT_KEY)
bgneal@522 132 if result == 1:
bgneal@522 133 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 134
bgneal@522 135 count = Post.objects.filter(topic__forum__in=public_forums).count()
bgneal@522 136 redis.set(POST_COUNT_KEY, count)
bgneal@522 137
bgneal@522 138
bgneal@522 139 def _update_latest_topics(redis, post):
bgneal@522 140 """
bgneal@522 141 Updates the "latest topics with new posts" list we cache in Redis for speed.
bgneal@522 142 There is a template tag and forum view that uses this information.
bgneal@522 143
bgneal@522 144 """
bgneal@522 145 # serialize topic attributes
bgneal@522 146 topic_id = post.topic.id
bgneal@522 147 topic_score = int(time.mktime(post.creation_date.timetuple()))
bgneal@522 148
bgneal@522 149 topic_content = {
bgneal@522 150 'title': post.topic.name,
bgneal@522 151 'author': post.user.username,
bgneal@522 152 'date': topic_score,
bgneal@522 153 'url': post.get_absolute_url()
bgneal@522 154 }
bgneal@522 155 json = simplejson.dumps(topic_content)
bgneal@522 156 key = UPDATED_TOPIC_KEY % topic_id
bgneal@522 157
bgneal@522 158 pipeline = redis.pipeline()
bgneal@522 159 pipeline.set(key, json)
bgneal@522 160 pipeline.zadd(UPDATED_TOPICS_SET_KEY, topic_score, topic_id)
bgneal@522 161 pipeline.zcard(UPDATED_TOPICS_SET_KEY)
bgneal@522 162 results = pipeline.execute()
bgneal@522 163
bgneal@522 164 # delete topics beyond our maximum count
bgneal@522 165 num_topics = results[-1]
bgneal@522 166 num_to_del = num_topics - MAX_UPDATED_TOPICS
bgneal@522 167 if num_to_del > 0:
bgneal@522 168 # get the IDs of the topics we need to delete first
bgneal@522 169 start = 0
bgneal@522 170 stop = num_to_del - 1 # Redis indices are inclusive
bgneal@522 171 old_ids = redis.zrange(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 172
bgneal@522 173 keys = [UPDATED_TOPIC_KEY % n for n in old_ids]
bgneal@522 174 redis.delete(*keys)
bgneal@522 175
bgneal@522 176 # now delete the oldest num_to_del topics
bgneal@522 177 redis.zremrangebyrank(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 178
bgneal@522 179
bgneal@509 180 def get_latest_posts(num_posts=MAX_POSTS, forum_id=None):
bgneal@509 181 """
bgneal@509 182 This function retrieves num_posts latest posts for the forum with the given
bgneal@509 183 forum_id. If forum_id is None, the posts are retrieved from the combined
bgneal@509 184 forums datastore. A list of dictionaries is returned. Each dictionary
bgneal@509 185 contains information about a post.
bgneal@509 186
bgneal@509 187 """
bgneal@509 188 key = 'forums:latest:%d' % forum_id if forum_id else 'forums:latest:*'
bgneal@509 189
bgneal@509 190 num_posts = max(0, min(MAX_POSTS, num_posts))
bgneal@509 191
bgneal@509 192 if num_posts == 0:
bgneal@509 193 return []
bgneal@509 194
bgneal@509 195 redis = get_redis_connection()
bgneal@509 196 raw_posts = redis.lrange(key, 0, num_posts - 1)
bgneal@509 197
bgneal@509 198 posts = []
bgneal@509 199 for raw_post in raw_posts:
bgneal@509 200 post = simplejson.loads(raw_post)
bgneal@509 201
bgneal@509 202 # fix up the pubdate; turn it back into a datetime object
bgneal@509 203 post['pubdate'] = datetime.datetime.fromtimestamp(post['pubdate'])
bgneal@509 204
bgneal@509 205 posts.append(post)
bgneal@509 206
bgneal@509 207 return posts
bgneal@522 208
bgneal@522 209
bgneal@522 210 @receiver(topic_content_update, dispatch_uid='forums.latest_posts')
bgneal@522 211 def on_topic_update(sender, **kwargs):
bgneal@522 212 """
bgneal@522 213 This function is our signal handler, called when a topic has been updated.
bgneal@522 214 We only care about newly created topics, and ignore updates.
bgneal@522 215
bgneal@522 216 We kick off a Celery task to perform work outside of the request/response
bgneal@522 217 cycle.
bgneal@522 218
bgneal@522 219 """
bgneal@522 220 # ignore non-new topics
bgneal@522 221 if not kwargs['created']:
bgneal@522 222 return
bgneal@522 223
bgneal@522 224 # Kick off a Celery task to process this new post
bgneal@522 225 forums.tasks.new_topic_task.delay(sender.id)
bgneal@522 226
bgneal@522 227
bgneal@522 228 def process_new_topic(topic_id):
bgneal@522 229 """
bgneal@522 230 This function contains new topic processing. Currently we only update the
bgneal@522 231 topic count statistic.
bgneal@522 232
bgneal@522 233 """
bgneal@522 234 try:
bgneal@522 235 topic = Topic.objects.select_related().get(pk=topic_id)
bgneal@522 236 except Topic.DoesNotExist:
bgneal@522 237 logger.warning("process_new_topic: topic %d does not exist", topic_id)
bgneal@522 238 return
bgneal@522 239
bgneal@522 240 # selectively process topics from non-public forums
bgneal@522 241 public_forums = Forum.objects.public_forum_ids()
bgneal@522 242
bgneal@522 243 if topic.forum.id not in public_forums:
bgneal@522 244 return
bgneal@522 245
bgneal@522 246 # update the topic count statistic
bgneal@522 247 redis = get_redis_connection()
bgneal@522 248
bgneal@522 249 result = redis.incr(TOPIC_COUNT_KEY)
bgneal@522 250 if result == 1:
bgneal@522 251 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 252
bgneal@522 253 count = Topic.objects.filter(forum__in=public_forums).count()
bgneal@522 254 redis.set(TOPIC_COUNT_KEY, count)
bgneal@522 255
bgneal@522 256
bgneal@522 257 def get_stats():
bgneal@522 258 """
bgneal@522 259 This function returns the topic and post count statistics as a tuple, in
bgneal@522 260 that order. If a statistic is not available, its position in the tuple will
bgneal@522 261 be None.
bgneal@522 262
bgneal@522 263 """
bgneal@522 264 try:
bgneal@522 265 redis = get_redis_connection()
bgneal@522 266 result = redis.mget(TOPIC_COUNT_KEY, POST_COUNT_KEY)
bgneal@522 267 except redis.RedisError, e:
bgneal@522 268 logger.error(e)
bgneal@522 269 return (None, None)
bgneal@522 270
bgneal@522 271 topic_count = int(result[0]) if result[0] else None
bgneal@522 272 post_count = int(result[1]) if result[1] else None
bgneal@522 273
bgneal@522 274 return (topic_count, post_count)
bgneal@522 275
bgneal@522 276
bgneal@522 277 def get_latest_topic_ids(num):
bgneal@522 278 """
bgneal@522 279 Return a list of topic ids from the latest topics that have posts. The ids
bgneal@522 280 will be sorted from newest to oldest.
bgneal@522 281
bgneal@522 282 """
bgneal@522 283 try:
bgneal@522 284 redis = get_redis_connection()
bgneal@522 285 result = redis.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 286 except redis.RedisError, e:
bgneal@522 287 logger.error(e)
bgneal@522 288 return []
bgneal@522 289
bgneal@522 290 return [int(n) for n in result]
bgneal@522 291
bgneal@522 292
bgneal@522 293 def get_latest_topics(num):
bgneal@522 294 """
bgneal@522 295 Return a list of dictionaries with information about the latest topics that
bgneal@522 296 have updated posts. The topics are sorted from newest to oldest.
bgneal@522 297
bgneal@522 298 """
bgneal@522 299 try:
bgneal@522 300 redis = get_redis_connection()
bgneal@522 301 result = redis.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 302
bgneal@522 303 topic_keys = [UPDATED_TOPIC_KEY % n for n in result]
bgneal@522 304 json_list = redis.mget(topic_keys)
bgneal@522 305
bgneal@522 306 except redis.RedisError, e:
bgneal@522 307 logger.error(e)
bgneal@522 308 return []
bgneal@522 309
bgneal@522 310 topics = []
bgneal@522 311 for s in json_list:
bgneal@522 312 item = simplejson.loads(s)
bgneal@522 313 item['date'] = datetime.datetime.fromtimestamp(item['date'])
bgneal@522 314 topics.append(item)
bgneal@522 315
bgneal@522 316 return topics
bgneal@522 317
bgneal@522 318
bgneal@522 319 def notify_topic_delete(topic):
bgneal@522 320 """
bgneal@522 321 This function should be called when a topic is deleted. It will remove the
bgneal@522 322 topic from the updated topics set, if present, and delete any info we have
bgneal@522 323 about the topic.
bgneal@522 324
bgneal@522 325 Note we don't do anything like this for posts. Since they just populate RSS
bgneal@522 326 feeds we'll let them 404. The updated topic list is seen in a prominent
bgneal@522 327 template tag however, so it is a bit more important to get that cleaned up.
bgneal@522 328
bgneal@522 329 """
bgneal@522 330 try:
bgneal@522 331 redis = get_redis_connection()
bgneal@522 332 pipeline = redis.pipeline()
bgneal@522 333 pipeline.zrem(UPDATED_TOPICS_SET_KEY, topic.id)
bgneal@522 334 pipeline.delete(UPDATED_TOPIC_KEY % topic.id)
bgneal@522 335 pipeline.execute()
bgneal@522 336 except redis.RedisError, e:
bgneal@522 337 logger.error(e)
bgneal@522 338
bgneal@522 339
bgneal@522 340 # Down here to avoid a circular import
bgneal@522 341 import forums.tasks