Mercurial > public > sg101

annotate forums/latest.py @ 594:2469d5864249

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Include links to video attachments in forum post RSS feeds. This is for bitbucket issue #9.
author Brian Neal <bgneal@gmail.com>
date Tue, 22 May 2012 19:53:39 -0500
parents ee87ea74d46b
children f3fded5df64b
rev   line source
bgneal@509 1 """
bgneal@509 2 This module maintains the latest posts datastore. The latest posts are often
bgneal@509 3 needed by RSS feeds, "latest posts" template tags, etc. This module listens for
bgneal@509 4 the post_content_update signal, then bundles the post up and stores it by forum
bgneal@509 5 ID in Redis. We also maintain a combined forums list. This allows quick
bgneal@509 6 retrieval of the latest posts and avoids some slow SQL queries.
bgneal@509 7
bgneal@522 8 We also do things like send topic notification emails, auto-favorite, and
bgneal@522 9 auto-subscribe functions here rather than bog the user down in the request /
bgneal@522 10 response cycle.
bgneal@522 11
bgneal@509 12 """
bgneal@509 13 import datetime
bgneal@522 14 import logging
bgneal@509 15 import time
bgneal@509 16
bgneal@509 17 from django.dispatch import receiver
bgneal@509 18 from django.utils import simplejson
bgneal@594 19 from django.template.loader import render_to_string
bgneal@523 20 import redis
bgneal@509 21
bgneal@522 22 from forums.signals import post_content_update, topic_content_update
bgneal@594 23 from forums.models import Forum, Topic, Post, Attachment
bgneal@522 24 from forums.views.subscriptions import notify_topic_subscribers
bgneal@522 25 from forums.tools import auto_favorite, auto_subscribe
bgneal@509 26 from core.services import get_redis_connection
bgneal@509 27
bgneal@509 28 # This constant controls how many latest posts per forum we store
bgneal@509 29 MAX_POSTS = 50
bgneal@509 30
bgneal@522 31 # This controls how many updated topics we track
bgneal@522 32 MAX_UPDATED_TOPICS = 50
bgneal@522 33
bgneal@522 34 # Redis key names:
bgneal@522 35 POST_COUNT_KEY = "forums:public_post_count"
bgneal@522 36 TOPIC_COUNT_KEY = "forums:public_topic_count"
bgneal@522 37 UPDATED_TOPICS_SET_KEY = "forums:updated_topics:set"
bgneal@522 38 UPDATED_TOPIC_KEY = "forums:updated_topics:%s"
bgneal@522 39
bgneal@522 40 logger = logging.getLogger(__name__)
bgneal@522 41
bgneal@509 42
bgneal@509 43 @receiver(post_content_update, dispatch_uid='forums.latest_posts')
bgneal@509 44 def on_post_update(sender, **kwargs):
bgneal@509 45 """
bgneal@509 46 This function is our signal handler, called when a post has been updated.
bgneal@509 47 We only care about newly created posts, and ignore updates.
bgneal@509 48
bgneal@522 49 We kick off a Celery task to perform work outside of the request/response
bgneal@522 50 cycle.
bgneal@509 51
bgneal@509 52 """
bgneal@509 53 # ignore non-new posts
bgneal@509 54 if not kwargs['created']:
bgneal@509 55 return
bgneal@509 56
bgneal@522 57 # Kick off a Celery task to process this new post
bgneal@522 58 forums.tasks.new_post_task.delay(sender.id)
bgneal@522 59
bgneal@522 60
bgneal@522 61 def process_new_post(post_id):
bgneal@522 62 """
bgneal@522 63 This function is run on a Celery task. It performs all new-post processing.
bgneal@522 64
bgneal@522 65 """
bgneal@522 66 try:
bgneal@522 67 post = Post.objects.select_related().get(pk=post_id)
bgneal@522 68 except Post.DoesNotExist:
bgneal@522 69 logger.warning("process_new_post: post %d does not exist", post_id)
bgneal@509 70 return
bgneal@509 71
bgneal@522 72 # selectively process posts from non-public forums
bgneal@522 73 public_forums = Forum.objects.public_forum_ids()
bgneal@522 74
bgneal@522 75 if post.topic.forum.id in public_forums:
bgneal@523 76 conn = get_redis_connection()
bgneal@523 77 _update_post_feeds(conn, post)
bgneal@523 78 _update_post_count(conn, public_forums)
bgneal@523 79 _update_latest_topics(conn, post)
bgneal@522 80
bgneal@522 81 # send out any email notifications
bgneal@522 82 notify_topic_subscribers(post, defer=False)
bgneal@522 83
bgneal@522 84 # perform any auto-favorite and auto-subscribe actions for the new post
bgneal@522 85 auto_favorite(post)
bgneal@522 86 auto_subscribe(post)
bgneal@522 87
bgneal@522 88
bgneal@523 89 def _update_post_feeds(conn, post):
bgneal@522 90 """
bgneal@522 91 Updates the forum feeds we keep in Redis so that our RSS feeds are quick.
bgneal@522 92
bgneal@522 93 """
bgneal@594 94 # get any attachments for the post
bgneal@594 95
bgneal@594 96 attachments = Attachment.objects.filter(post=post).select_related(
bgneal@594 97 'embed').order_by('order')
bgneal@594 98 embeds = [item.embed for item in attachments]
bgneal@594 99 if len(embeds) == 0:
bgneal@594 100 content = post.html
bgneal@594 101 else:
bgneal@594 102 content = render_to_string('forums/post_rss.html', {
bgneal@594 103 'post': post,
bgneal@594 104 'embeds': embeds,
bgneal@594 105 })
bgneal@594 106
bgneal@509 107 # serialize post attributes
bgneal@509 108 post_content = {
bgneal@522 109 'id': post.id,
bgneal@522 110 'title': post.topic.name,
bgneal@594 111 'content': content,
bgneal@522 112 'author': post.user.username,
bgneal@522 113 'pubdate': int(time.mktime(post.creation_date.timetuple())),
bgneal@522 114 'forum_name': post.topic.forum.name,
bgneal@522 115 'url': post.get_absolute_url()
bgneal@509 116 }
bgneal@509 117
bgneal@509 118 s = simplejson.dumps(post_content)
bgneal@509 119
bgneal@509 120 # store in Redis
bgneal@509 121
bgneal@523 122 pipeline = conn.pipeline()
bgneal@509 123
bgneal@522 124 key = 'forums:latest:%d' % post.topic.forum.id
bgneal@509 125
bgneal@509 126 pipeline.lpush(key, s)
bgneal@509 127 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 128
bgneal@509 129 # store in the combined feed; yes this wastes some memory storing it twice,
bgneal@509 130 # but it makes things much easier
bgneal@509 131
bgneal@509 132 key = 'forums:latest:*'
bgneal@509 133
bgneal@509 134 pipeline.lpush(key, s)
bgneal@509 135 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 136
bgneal@509 137 pipeline.execute()
bgneal@509 138
bgneal@509 139
bgneal@523 140 def _update_post_count(conn, public_forums):
bgneal@522 141 """
bgneal@522 142 Updates the post count we cache in Redis. Doing a COUNT(*) on the post table
bgneal@522 143 can be expensive in MySQL InnoDB.
bgneal@522 144
bgneal@522 145 """
bgneal@523 146 result = conn.incr(POST_COUNT_KEY)
bgneal@522 147 if result == 1:
bgneal@522 148 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 149
bgneal@522 150 count = Post.objects.filter(topic__forum__in=public_forums).count()
bgneal@523 151 conn.set(POST_COUNT_KEY, count)
bgneal@522 152
bgneal@522 153
bgneal@523 154 def _update_latest_topics(conn, post):
bgneal@522 155 """
bgneal@522 156 Updates the "latest topics with new posts" list we cache in Redis for speed.
bgneal@522 157 There is a template tag and forum view that uses this information.
bgneal@522 158
bgneal@522 159 """
bgneal@522 160 # serialize topic attributes
bgneal@522 161 topic_id = post.topic.id
bgneal@522 162 topic_score = int(time.mktime(post.creation_date.timetuple()))
bgneal@522 163
bgneal@522 164 topic_content = {
bgneal@522 165 'title': post.topic.name,
bgneal@522 166 'author': post.user.username,
bgneal@522 167 'date': topic_score,
bgneal@529 168 'url': post.topic.get_latest_post_url()
bgneal@522 169 }
bgneal@522 170 json = simplejson.dumps(topic_content)
bgneal@522 171 key = UPDATED_TOPIC_KEY % topic_id
bgneal@522 172
bgneal@523 173 pipeline = conn.pipeline()
bgneal@522 174 pipeline.set(key, json)
bgneal@522 175 pipeline.zadd(UPDATED_TOPICS_SET_KEY, topic_score, topic_id)
bgneal@522 176 pipeline.zcard(UPDATED_TOPICS_SET_KEY)
bgneal@522 177 results = pipeline.execute()
bgneal@522 178
bgneal@522 179 # delete topics beyond our maximum count
bgneal@522 180 num_topics = results[-1]
bgneal@522 181 num_to_del = num_topics - MAX_UPDATED_TOPICS
bgneal@522 182 if num_to_del > 0:
bgneal@522 183 # get the IDs of the topics we need to delete first
bgneal@522 184 start = 0
bgneal@522 185 stop = num_to_del - 1 # Redis indices are inclusive
bgneal@523 186 old_ids = conn.zrange(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 187
bgneal@522 188 keys = [UPDATED_TOPIC_KEY % n for n in old_ids]
bgneal@523 189 conn.delete(*keys)
bgneal@522 190
bgneal@522 191 # now delete the oldest num_to_del topics
bgneal@523 192 conn.zremrangebyrank(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 193
bgneal@522 194
bgneal@509 195 def get_latest_posts(num_posts=MAX_POSTS, forum_id=None):
bgneal@509 196 """
bgneal@509 197 This function retrieves num_posts latest posts for the forum with the given
bgneal@509 198 forum_id. If forum_id is None, the posts are retrieved from the combined
bgneal@509 199 forums datastore. A list of dictionaries is returned. Each dictionary
bgneal@509 200 contains information about a post.
bgneal@509 201
bgneal@509 202 """
bgneal@509 203 key = 'forums:latest:%d' % forum_id if forum_id else 'forums:latest:*'
bgneal@509 204
bgneal@509 205 num_posts = max(0, min(MAX_POSTS, num_posts))
bgneal@509 206
bgneal@509 207 if num_posts == 0:
bgneal@509 208 return []
bgneal@509 209
bgneal@523 210 conn = get_redis_connection()
bgneal@523 211 raw_posts = conn.lrange(key, 0, num_posts - 1)
bgneal@509 212
bgneal@509 213 posts = []
bgneal@509 214 for raw_post in raw_posts:
bgneal@509 215 post = simplejson.loads(raw_post)
bgneal@509 216
bgneal@509 217 # fix up the pubdate; turn it back into a datetime object
bgneal@509 218 post['pubdate'] = datetime.datetime.fromtimestamp(post['pubdate'])
bgneal@509 219
bgneal@509 220 posts.append(post)
bgneal@509 221
bgneal@509 222 return posts
bgneal@522 223
bgneal@522 224
bgneal@522 225 @receiver(topic_content_update, dispatch_uid='forums.latest_posts')
bgneal@522 226 def on_topic_update(sender, **kwargs):
bgneal@522 227 """
bgneal@522 228 This function is our signal handler, called when a topic has been updated.
bgneal@522 229 We only care about newly created topics, and ignore updates.
bgneal@522 230
bgneal@522 231 We kick off a Celery task to perform work outside of the request/response
bgneal@522 232 cycle.
bgneal@522 233
bgneal@522 234 """
bgneal@522 235 # ignore non-new topics
bgneal@522 236 if not kwargs['created']:
bgneal@522 237 return
bgneal@522 238
bgneal@522 239 # Kick off a Celery task to process this new post
bgneal@522 240 forums.tasks.new_topic_task.delay(sender.id)
bgneal@522 241
bgneal@522 242
bgneal@522 243 def process_new_topic(topic_id):
bgneal@522 244 """
bgneal@522 245 This function contains new topic processing. Currently we only update the
bgneal@522 246 topic count statistic.
bgneal@522 247
bgneal@522 248 """
bgneal@522 249 try:
bgneal@522 250 topic = Topic.objects.select_related().get(pk=topic_id)
bgneal@522 251 except Topic.DoesNotExist:
bgneal@522 252 logger.warning("process_new_topic: topic %d does not exist", topic_id)
bgneal@522 253 return
bgneal@522 254
bgneal@522 255 # selectively process topics from non-public forums
bgneal@522 256 public_forums = Forum.objects.public_forum_ids()
bgneal@522 257
bgneal@522 258 if topic.forum.id not in public_forums:
bgneal@522 259 return
bgneal@522 260
bgneal@522 261 # update the topic count statistic
bgneal@523 262 conn = get_redis_connection()
bgneal@522 263
bgneal@523 264 result = conn.incr(TOPIC_COUNT_KEY)
bgneal@522 265 if result == 1:
bgneal@522 266 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 267
bgneal@522 268 count = Topic.objects.filter(forum__in=public_forums).count()
bgneal@523 269 conn.set(TOPIC_COUNT_KEY, count)
bgneal@522 270
bgneal@522 271
bgneal@522 272 def get_stats():
bgneal@522 273 """
bgneal@522 274 This function returns the topic and post count statistics as a tuple, in
bgneal@522 275 that order. If a statistic is not available, its position in the tuple will
bgneal@522 276 be None.
bgneal@522 277
bgneal@522 278 """
bgneal@522 279 try:
bgneal@523 280 conn = get_redis_connection()
bgneal@523 281 result = conn.mget(TOPIC_COUNT_KEY, POST_COUNT_KEY)
bgneal@522 282 except redis.RedisError, e:
bgneal@522 283 logger.error(e)
bgneal@522 284 return (None, None)
bgneal@522 285
bgneal@522 286 topic_count = int(result[0]) if result[0] else None
bgneal@522 287 post_count = int(result[1]) if result[1] else None
bgneal@522 288
bgneal@522 289 return (topic_count, post_count)
bgneal@522 290
bgneal@522 291
bgneal@522 292 def get_latest_topic_ids(num):
bgneal@522 293 """
bgneal@522 294 Return a list of topic ids from the latest topics that have posts. The ids
bgneal@522 295 will be sorted from newest to oldest.
bgneal@522 296
bgneal@522 297 """
bgneal@522 298 try:
bgneal@523 299 conn = get_redis_connection()
bgneal@523 300 result = conn.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 301 except redis.RedisError, e:
bgneal@522 302 logger.error(e)
bgneal@522 303 return []
bgneal@522 304
bgneal@522 305 return [int(n) for n in result]
bgneal@522 306
bgneal@522 307
bgneal@522 308 def get_latest_topics(num):
bgneal@522 309 """
bgneal@522 310 Return a list of dictionaries with information about the latest topics that
bgneal@522 311 have updated posts. The topics are sorted from newest to oldest.
bgneal@522 312
bgneal@522 313 """
bgneal@522 314 try:
bgneal@523 315 conn = get_redis_connection()
bgneal@523 316 result = conn.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 317
bgneal@522 318 topic_keys = [UPDATED_TOPIC_KEY % n for n in result]
bgneal@524 319 json_list = conn.mget(topic_keys) if topic_keys else []
bgneal@522 320
bgneal@522 321 except redis.RedisError, e:
bgneal@522 322 logger.error(e)
bgneal@522 323 return []
bgneal@522 324
bgneal@522 325 topics = []
bgneal@522 326 for s in json_list:
bgneal@522 327 item = simplejson.loads(s)
bgneal@522 328 item['date'] = datetime.datetime.fromtimestamp(item['date'])
bgneal@522 329 topics.append(item)
bgneal@522 330
bgneal@522 331 return topics
bgneal@522 332
bgneal@522 333
bgneal@522 334 def notify_topic_delete(topic):
bgneal@522 335 """
bgneal@522 336 This function should be called when a topic is deleted. It will remove the
bgneal@522 337 topic from the updated topics set, if present, and delete any info we have
bgneal@522 338 about the topic.
bgneal@522 339
bgneal@522 340 Note we don't do anything like this for posts. Since they just populate RSS
bgneal@522 341 feeds we'll let them 404. The updated topic list is seen in a prominent
bgneal@522 342 template tag however, so it is a bit more important to get that cleaned up.
bgneal@522 343
bgneal@522 344 """
bgneal@522 345 try:
bgneal@523 346 conn = get_redis_connection()
bgneal@523 347 pipeline = conn.pipeline()
bgneal@522 348 pipeline.zrem(UPDATED_TOPICS_SET_KEY, topic.id)
bgneal@522 349 pipeline.delete(UPDATED_TOPIC_KEY % topic.id)
bgneal@522 350 pipeline.execute()
bgneal@522 351 except redis.RedisError, e:
bgneal@522 352 logger.error(e)
bgneal@522 353
bgneal@522 354
bgneal@522 355 # Down here to avoid a circular import
bgneal@522 356 import forums.tasks