Mercurial > public > sg101

annotate gpp/forums/latest.py @ 523:e9c446a64423

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Was catching redis.RedisError exception, but already had local variable called redis. This seemed to work in development, but failed on Python 2.5.
author Brian Neal <bgneal@gmail.com>
date Sun, 18 Dec 2011 23:55:53 +0000
parents 82b97697312e
children d9cf6f60b5a5
rev   line source
bgneal@509 1 """
bgneal@509 2 This module maintains the latest posts datastore. The latest posts are often
bgneal@509 3 needed by RSS feeds, "latest posts" template tags, etc. This module listens for
bgneal@509 4 the post_content_update signal, then bundles the post up and stores it by forum
bgneal@509 5 ID in Redis. We also maintain a combined forums list. This allows quick
bgneal@509 6 retrieval of the latest posts and avoids some slow SQL queries.
bgneal@509 7
bgneal@522 8 We also do things like send topic notification emails, auto-favorite, and
bgneal@522 9 auto-subscribe functions here rather than bog the user down in the request /
bgneal@522 10 response cycle.
bgneal@522 11
bgneal@509 12 """
bgneal@509 13 import datetime
bgneal@522 14 import logging
bgneal@509 15 import time
bgneal@509 16
bgneal@509 17 from django.dispatch import receiver
bgneal@509 18 from django.utils import simplejson
bgneal@523 19 import redis
bgneal@509 20
bgneal@522 21 from forums.signals import post_content_update, topic_content_update
bgneal@522 22 from forums.models import Forum, Topic, Post
bgneal@522 23 from forums.views.subscriptions import notify_topic_subscribers
bgneal@522 24 from forums.tools import auto_favorite, auto_subscribe
bgneal@509 25 from core.services import get_redis_connection
bgneal@509 26
bgneal@509 27 # This constant controls how many latest posts per forum we store
bgneal@509 28 MAX_POSTS = 50
bgneal@509 29
bgneal@522 30 # This controls how many updated topics we track
bgneal@522 31 MAX_UPDATED_TOPICS = 50
bgneal@522 32
bgneal@522 33 # Redis key names:
bgneal@522 34 POST_COUNT_KEY = "forums:public_post_count"
bgneal@522 35 TOPIC_COUNT_KEY = "forums:public_topic_count"
bgneal@522 36 UPDATED_TOPICS_SET_KEY = "forums:updated_topics:set"
bgneal@522 37 UPDATED_TOPIC_KEY = "forums:updated_topics:%s"
bgneal@522 38
bgneal@522 39 logger = logging.getLogger(__name__)
bgneal@522 40
bgneal@509 41
bgneal@509 42 @receiver(post_content_update, dispatch_uid='forums.latest_posts')
bgneal@509 43 def on_post_update(sender, **kwargs):
bgneal@509 44 """
bgneal@509 45 This function is our signal handler, called when a post has been updated.
bgneal@509 46 We only care about newly created posts, and ignore updates.
bgneal@509 47
bgneal@522 48 We kick off a Celery task to perform work outside of the request/response
bgneal@522 49 cycle.
bgneal@509 50
bgneal@509 51 """
bgneal@509 52 # ignore non-new posts
bgneal@509 53 if not kwargs['created']:
bgneal@509 54 return
bgneal@509 55
bgneal@522 56 # Kick off a Celery task to process this new post
bgneal@522 57 forums.tasks.new_post_task.delay(sender.id)
bgneal@522 58
bgneal@522 59
bgneal@522 60 def process_new_post(post_id):
bgneal@522 61 """
bgneal@522 62 This function is run on a Celery task. It performs all new-post processing.
bgneal@522 63
bgneal@522 64 """
bgneal@522 65 try:
bgneal@522 66 post = Post.objects.select_related().get(pk=post_id)
bgneal@522 67 except Post.DoesNotExist:
bgneal@522 68 logger.warning("process_new_post: post %d does not exist", post_id)
bgneal@509 69 return
bgneal@509 70
bgneal@522 71 # selectively process posts from non-public forums
bgneal@522 72 public_forums = Forum.objects.public_forum_ids()
bgneal@522 73
bgneal@522 74 if post.topic.forum.id in public_forums:
bgneal@523 75 conn = get_redis_connection()
bgneal@523 76 _update_post_feeds(conn, post)
bgneal@523 77 _update_post_count(conn, public_forums)
bgneal@523 78 _update_latest_topics(conn, post)
bgneal@522 79
bgneal@522 80 # send out any email notifications
bgneal@522 81 notify_topic_subscribers(post, defer=False)
bgneal@522 82
bgneal@522 83 # perform any auto-favorite and auto-subscribe actions for the new post
bgneal@522 84 auto_favorite(post)
bgneal@522 85 auto_subscribe(post)
bgneal@522 86
bgneal@522 87
bgneal@523 88 def _update_post_feeds(conn, post):
bgneal@522 89 """
bgneal@522 90 Updates the forum feeds we keep in Redis so that our RSS feeds are quick.
bgneal@522 91
bgneal@522 92 """
bgneal@509 93 # serialize post attributes
bgneal@509 94 post_content = {
bgneal@522 95 'id': post.id,
bgneal@522 96 'title': post.topic.name,
bgneal@522 97 'content': post.html,
bgneal@522 98 'author': post.user.username,
bgneal@522 99 'pubdate': int(time.mktime(post.creation_date.timetuple())),
bgneal@522 100 'forum_name': post.topic.forum.name,
bgneal@522 101 'url': post.get_absolute_url()
bgneal@509 102 }
bgneal@509 103
bgneal@509 104 s = simplejson.dumps(post_content)
bgneal@509 105
bgneal@509 106 # store in Redis
bgneal@509 107
bgneal@523 108 pipeline = conn.pipeline()
bgneal@509 109
bgneal@522 110 key = 'forums:latest:%d' % post.topic.forum.id
bgneal@509 111
bgneal@509 112 pipeline.lpush(key, s)
bgneal@509 113 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 114
bgneal@509 115 # store in the combined feed; yes this wastes some memory storing it twice,
bgneal@509 116 # but it makes things much easier
bgneal@509 117
bgneal@509 118 key = 'forums:latest:*'
bgneal@509 119
bgneal@509 120 pipeline.lpush(key, s)
bgneal@509 121 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 122
bgneal@509 123 pipeline.execute()
bgneal@509 124
bgneal@509 125
bgneal@523 126 def _update_post_count(conn, public_forums):
bgneal@522 127 """
bgneal@522 128 Updates the post count we cache in Redis. Doing a COUNT(*) on the post table
bgneal@522 129 can be expensive in MySQL InnoDB.
bgneal@522 130
bgneal@522 131 """
bgneal@523 132 result = conn.incr(POST_COUNT_KEY)
bgneal@522 133 if result == 1:
bgneal@522 134 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 135
bgneal@522 136 count = Post.objects.filter(topic__forum__in=public_forums).count()
bgneal@523 137 conn.set(POST_COUNT_KEY, count)
bgneal@522 138
bgneal@522 139
bgneal@523 140 def _update_latest_topics(conn, post):
bgneal@522 141 """
bgneal@522 142 Updates the "latest topics with new posts" list we cache in Redis for speed.
bgneal@522 143 There is a template tag and forum view that uses this information.
bgneal@522 144
bgneal@522 145 """
bgneal@522 146 # serialize topic attributes
bgneal@522 147 topic_id = post.topic.id
bgneal@522 148 topic_score = int(time.mktime(post.creation_date.timetuple()))
bgneal@522 149
bgneal@522 150 topic_content = {
bgneal@522 151 'title': post.topic.name,
bgneal@522 152 'author': post.user.username,
bgneal@522 153 'date': topic_score,
bgneal@522 154 'url': post.get_absolute_url()
bgneal@522 155 }
bgneal@522 156 json = simplejson.dumps(topic_content)
bgneal@522 157 key = UPDATED_TOPIC_KEY % topic_id
bgneal@522 158
bgneal@523 159 pipeline = conn.pipeline()
bgneal@522 160 pipeline.set(key, json)
bgneal@522 161 pipeline.zadd(UPDATED_TOPICS_SET_KEY, topic_score, topic_id)
bgneal@522 162 pipeline.zcard(UPDATED_TOPICS_SET_KEY)
bgneal@522 163 results = pipeline.execute()
bgneal@522 164
bgneal@522 165 # delete topics beyond our maximum count
bgneal@522 166 num_topics = results[-1]
bgneal@522 167 num_to_del = num_topics - MAX_UPDATED_TOPICS
bgneal@522 168 if num_to_del > 0:
bgneal@522 169 # get the IDs of the topics we need to delete first
bgneal@522 170 start = 0
bgneal@522 171 stop = num_to_del - 1 # Redis indices are inclusive
bgneal@523 172 old_ids = conn.zrange(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 173
bgneal@522 174 keys = [UPDATED_TOPIC_KEY % n for n in old_ids]
bgneal@523 175 conn.delete(*keys)
bgneal@522 176
bgneal@522 177 # now delete the oldest num_to_del topics
bgneal@523 178 conn.zremrangebyrank(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 179
bgneal@522 180
bgneal@509 181 def get_latest_posts(num_posts=MAX_POSTS, forum_id=None):
bgneal@509 182 """
bgneal@509 183 This function retrieves num_posts latest posts for the forum with the given
bgneal@509 184 forum_id. If forum_id is None, the posts are retrieved from the combined
bgneal@509 185 forums datastore. A list of dictionaries is returned. Each dictionary
bgneal@509 186 contains information about a post.
bgneal@509 187
bgneal@509 188 """
bgneal@509 189 key = 'forums:latest:%d' % forum_id if forum_id else 'forums:latest:*'
bgneal@509 190
bgneal@509 191 num_posts = max(0, min(MAX_POSTS, num_posts))
bgneal@509 192
bgneal@509 193 if num_posts == 0:
bgneal@509 194 return []
bgneal@509 195
bgneal@523 196 conn = get_redis_connection()
bgneal@523 197 raw_posts = conn.lrange(key, 0, num_posts - 1)
bgneal@509 198
bgneal@509 199 posts = []
bgneal@509 200 for raw_post in raw_posts:
bgneal@509 201 post = simplejson.loads(raw_post)
bgneal@509 202
bgneal@509 203 # fix up the pubdate; turn it back into a datetime object
bgneal@509 204 post['pubdate'] = datetime.datetime.fromtimestamp(post['pubdate'])
bgneal@509 205
bgneal@509 206 posts.append(post)
bgneal@509 207
bgneal@509 208 return posts
bgneal@522 209
bgneal@522 210
bgneal@522 211 @receiver(topic_content_update, dispatch_uid='forums.latest_posts')
bgneal@522 212 def on_topic_update(sender, **kwargs):
bgneal@522 213 """
bgneal@522 214 This function is our signal handler, called when a topic has been updated.
bgneal@522 215 We only care about newly created topics, and ignore updates.
bgneal@522 216
bgneal@522 217 We kick off a Celery task to perform work outside of the request/response
bgneal@522 218 cycle.
bgneal@522 219
bgneal@522 220 """
bgneal@522 221 # ignore non-new topics
bgneal@522 222 if not kwargs['created']:
bgneal@522 223 return
bgneal@522 224
bgneal@522 225 # Kick off a Celery task to process this new post
bgneal@522 226 forums.tasks.new_topic_task.delay(sender.id)
bgneal@522 227
bgneal@522 228
bgneal@522 229 def process_new_topic(topic_id):
bgneal@522 230 """
bgneal@522 231 This function contains new topic processing. Currently we only update the
bgneal@522 232 topic count statistic.
bgneal@522 233
bgneal@522 234 """
bgneal@522 235 try:
bgneal@522 236 topic = Topic.objects.select_related().get(pk=topic_id)
bgneal@522 237 except Topic.DoesNotExist:
bgneal@522 238 logger.warning("process_new_topic: topic %d does not exist", topic_id)
bgneal@522 239 return
bgneal@522 240
bgneal@522 241 # selectively process topics from non-public forums
bgneal@522 242 public_forums = Forum.objects.public_forum_ids()
bgneal@522 243
bgneal@522 244 if topic.forum.id not in public_forums:
bgneal@522 245 return
bgneal@522 246
bgneal@522 247 # update the topic count statistic
bgneal@523 248 conn = get_redis_connection()
bgneal@522 249
bgneal@523 250 result = conn.incr(TOPIC_COUNT_KEY)
bgneal@522 251 if result == 1:
bgneal@522 252 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 253
bgneal@522 254 count = Topic.objects.filter(forum__in=public_forums).count()
bgneal@523 255 conn.set(TOPIC_COUNT_KEY, count)
bgneal@522 256
bgneal@522 257
bgneal@522 258 def get_stats():
bgneal@522 259 """
bgneal@522 260 This function returns the topic and post count statistics as a tuple, in
bgneal@522 261 that order. If a statistic is not available, its position in the tuple will
bgneal@522 262 be None.
bgneal@522 263
bgneal@522 264 """
bgneal@522 265 try:
bgneal@523 266 conn = get_redis_connection()
bgneal@523 267 result = conn.mget(TOPIC_COUNT_KEY, POST_COUNT_KEY)
bgneal@522 268 except redis.RedisError, e:
bgneal@522 269 logger.error(e)
bgneal@522 270 return (None, None)
bgneal@522 271
bgneal@522 272 topic_count = int(result[0]) if result[0] else None
bgneal@522 273 post_count = int(result[1]) if result[1] else None
bgneal@522 274
bgneal@522 275 return (topic_count, post_count)
bgneal@522 276
bgneal@522 277
bgneal@522 278 def get_latest_topic_ids(num):
bgneal@522 279 """
bgneal@522 280 Return a list of topic ids from the latest topics that have posts. The ids
bgneal@522 281 will be sorted from newest to oldest.
bgneal@522 282
bgneal@522 283 """
bgneal@522 284 try:
bgneal@523 285 conn = get_redis_connection()
bgneal@523 286 result = conn.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 287 except redis.RedisError, e:
bgneal@522 288 logger.error(e)
bgneal@522 289 return []
bgneal@522 290
bgneal@522 291 return [int(n) for n in result]
bgneal@522 292
bgneal@522 293
bgneal@522 294 def get_latest_topics(num):
bgneal@522 295 """
bgneal@522 296 Return a list of dictionaries with information about the latest topics that
bgneal@522 297 have updated posts. The topics are sorted from newest to oldest.
bgneal@522 298
bgneal@522 299 """
bgneal@522 300 try:
bgneal@523 301 conn = get_redis_connection()
bgneal@523 302 result = conn.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 303
bgneal@522 304 topic_keys = [UPDATED_TOPIC_KEY % n for n in result]
bgneal@523 305 json_list = conn.mget(topic_keys)
bgneal@522 306
bgneal@522 307 except redis.RedisError, e:
bgneal@522 308 logger.error(e)
bgneal@522 309 return []
bgneal@522 310
bgneal@522 311 topics = []
bgneal@522 312 for s in json_list:
bgneal@522 313 item = simplejson.loads(s)
bgneal@522 314 item['date'] = datetime.datetime.fromtimestamp(item['date'])
bgneal@522 315 topics.append(item)
bgneal@522 316
bgneal@522 317 return topics
bgneal@522 318
bgneal@522 319
bgneal@522 320 def notify_topic_delete(topic):
bgneal@522 321 """
bgneal@522 322 This function should be called when a topic is deleted. It will remove the
bgneal@522 323 topic from the updated topics set, if present, and delete any info we have
bgneal@522 324 about the topic.
bgneal@522 325
bgneal@522 326 Note we don't do anything like this for posts. Since they just populate RSS
bgneal@522 327 feeds we'll let them 404. The updated topic list is seen in a prominent
bgneal@522 328 template tag however, so it is a bit more important to get that cleaned up.
bgneal@522 329
bgneal@522 330 """
bgneal@522 331 try:
bgneal@523 332 conn = get_redis_connection()
bgneal@523 333 pipeline = conn.pipeline()
bgneal@522 334 pipeline.zrem(UPDATED_TOPICS_SET_KEY, topic.id)
bgneal@522 335 pipeline.delete(UPDATED_TOPIC_KEY % topic.id)
bgneal@522 336 pipeline.execute()
bgneal@522 337 except redis.RedisError, e:
bgneal@522 338 logger.error(e)
bgneal@522 339
bgneal@522 340
bgneal@522 341 # Down here to avoid a circular import
bgneal@522 342 import forums.tasks