Mercurial > public > sg101

annotate forums/latest.py @ 582:392fd36360c6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added a requirements_dev.txt file since with Python 2.7 we no longer need the importlib.
author Brian Neal <bgneal@gmail.com>
date Sat, 05 May 2012 17:28:29 -0500
parents ee87ea74d46b
children 2469d5864249
rev   line source
bgneal@509 1 """
bgneal@509 2 This module maintains the latest posts datastore. The latest posts are often
bgneal@509 3 needed by RSS feeds, "latest posts" template tags, etc. This module listens for
bgneal@509 4 the post_content_update signal, then bundles the post up and stores it by forum
bgneal@509 5 ID in Redis. We also maintain a combined forums list. This allows quick
bgneal@509 6 retrieval of the latest posts and avoids some slow SQL queries.
bgneal@509 7
bgneal@522 8 We also do things like send topic notification emails, auto-favorite, and
bgneal@522 9 auto-subscribe functions here rather than bog the user down in the request /
bgneal@522 10 response cycle.
bgneal@522 11
bgneal@509 12 """
bgneal@509 13 import datetime
bgneal@522 14 import logging
bgneal@509 15 import time
bgneal@509 16
bgneal@509 17 from django.dispatch import receiver
bgneal@509 18 from django.utils import simplejson
bgneal@523 19 import redis
bgneal@509 20
bgneal@522 21 from forums.signals import post_content_update, topic_content_update
bgneal@522 22 from forums.models import Forum, Topic, Post
bgneal@522 23 from forums.views.subscriptions import notify_topic_subscribers
bgneal@522 24 from forums.tools import auto_favorite, auto_subscribe
bgneal@509 25 from core.services import get_redis_connection
bgneal@509 26
bgneal@509 27 # This constant controls how many latest posts per forum we store
bgneal@509 28 MAX_POSTS = 50
bgneal@509 29
bgneal@522 30 # This controls how many updated topics we track
bgneal@522 31 MAX_UPDATED_TOPICS = 50
bgneal@522 32
bgneal@522 33 # Redis key names:
bgneal@522 34 POST_COUNT_KEY = "forums:public_post_count"
bgneal@522 35 TOPIC_COUNT_KEY = "forums:public_topic_count"
bgneal@522 36 UPDATED_TOPICS_SET_KEY = "forums:updated_topics:set"
bgneal@522 37 UPDATED_TOPIC_KEY = "forums:updated_topics:%s"
bgneal@522 38
bgneal@522 39 logger = logging.getLogger(__name__)
bgneal@522 40
bgneal@509 41
bgneal@509 42 @receiver(post_content_update, dispatch_uid='forums.latest_posts')
bgneal@509 43 def on_post_update(sender, **kwargs):
bgneal@509 44 """
bgneal@509 45 This function is our signal handler, called when a post has been updated.
bgneal@509 46 We only care about newly created posts, and ignore updates.
bgneal@509 47
bgneal@522 48 We kick off a Celery task to perform work outside of the request/response
bgneal@522 49 cycle.
bgneal@509 50
bgneal@509 51 """
bgneal@509 52 # ignore non-new posts
bgneal@509 53 if not kwargs['created']:
bgneal@509 54 return
bgneal@509 55
bgneal@522 56 # Kick off a Celery task to process this new post
bgneal@522 57 forums.tasks.new_post_task.delay(sender.id)
bgneal@522 58
bgneal@522 59
bgneal@522 60 def process_new_post(post_id):
bgneal@522 61 """
bgneal@522 62 This function is run on a Celery task. It performs all new-post processing.
bgneal@522 63
bgneal@522 64 """
bgneal@522 65 try:
bgneal@522 66 post = Post.objects.select_related().get(pk=post_id)
bgneal@522 67 except Post.DoesNotExist:
bgneal@522 68 logger.warning("process_new_post: post %d does not exist", post_id)
bgneal@509 69 return
bgneal@509 70
bgneal@522 71 # selectively process posts from non-public forums
bgneal@522 72 public_forums = Forum.objects.public_forum_ids()
bgneal@522 73
bgneal@522 74 if post.topic.forum.id in public_forums:
bgneal@523 75 conn = get_redis_connection()
bgneal@523 76 _update_post_feeds(conn, post)
bgneal@523 77 _update_post_count(conn, public_forums)
bgneal@523 78 _update_latest_topics(conn, post)
bgneal@522 79
bgneal@522 80 # send out any email notifications
bgneal@522 81 notify_topic_subscribers(post, defer=False)
bgneal@522 82
bgneal@522 83 # perform any auto-favorite and auto-subscribe actions for the new post
bgneal@522 84 auto_favorite(post)
bgneal@522 85 auto_subscribe(post)
bgneal@522 86
bgneal@522 87
bgneal@523 88 def _update_post_feeds(conn, post):
bgneal@522 89 """
bgneal@522 90 Updates the forum feeds we keep in Redis so that our RSS feeds are quick.
bgneal@522 91
bgneal@522 92 """
bgneal@509 93 # serialize post attributes
bgneal@509 94 post_content = {
bgneal@522 95 'id': post.id,
bgneal@522 96 'title': post.topic.name,
bgneal@522 97 'content': post.html,
bgneal@522 98 'author': post.user.username,
bgneal@522 99 'pubdate': int(time.mktime(post.creation_date.timetuple())),
bgneal@522 100 'forum_name': post.topic.forum.name,
bgneal@522 101 'url': post.get_absolute_url()
bgneal@509 102 }
bgneal@509 103
bgneal@509 104 s = simplejson.dumps(post_content)
bgneal@509 105
bgneal@509 106 # store in Redis
bgneal@509 107
bgneal@523 108 pipeline = conn.pipeline()
bgneal@509 109
bgneal@522 110 key = 'forums:latest:%d' % post.topic.forum.id
bgneal@509 111
bgneal@509 112 pipeline.lpush(key, s)
bgneal@509 113 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 114
bgneal@509 115 # store in the combined feed; yes this wastes some memory storing it twice,
bgneal@509 116 # but it makes things much easier
bgneal@509 117
bgneal@509 118 key = 'forums:latest:*'
bgneal@509 119
bgneal@509 120 pipeline.lpush(key, s)
bgneal@509 121 pipeline.ltrim(key, 0, MAX_POSTS - 1)
bgneal@509 122
bgneal@509 123 pipeline.execute()
bgneal@509 124
bgneal@509 125
bgneal@523 126 def _update_post_count(conn, public_forums):
bgneal@522 127 """
bgneal@522 128 Updates the post count we cache in Redis. Doing a COUNT(*) on the post table
bgneal@522 129 can be expensive in MySQL InnoDB.
bgneal@522 130
bgneal@522 131 """
bgneal@523 132 result = conn.incr(POST_COUNT_KEY)
bgneal@522 133 if result == 1:
bgneal@522 134 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 135
bgneal@522 136 count = Post.objects.filter(topic__forum__in=public_forums).count()
bgneal@523 137 conn.set(POST_COUNT_KEY, count)
bgneal@522 138
bgneal@522 139
bgneal@523 140 def _update_latest_topics(conn, post):
bgneal@522 141 """
bgneal@522 142 Updates the "latest topics with new posts" list we cache in Redis for speed.
bgneal@522 143 There is a template tag and forum view that uses this information.
bgneal@522 144
bgneal@522 145 """
bgneal@522 146 # serialize topic attributes
bgneal@522 147 topic_id = post.topic.id
bgneal@522 148 topic_score = int(time.mktime(post.creation_date.timetuple()))
bgneal@522 149
bgneal@522 150 topic_content = {
bgneal@522 151 'title': post.topic.name,
bgneal@522 152 'author': post.user.username,
bgneal@522 153 'date': topic_score,
bgneal@529 154 'url': post.topic.get_latest_post_url()
bgneal@522 155 }
bgneal@522 156 json = simplejson.dumps(topic_content)
bgneal@522 157 key = UPDATED_TOPIC_KEY % topic_id
bgneal@522 158
bgneal@523 159 pipeline = conn.pipeline()
bgneal@522 160 pipeline.set(key, json)
bgneal@522 161 pipeline.zadd(UPDATED_TOPICS_SET_KEY, topic_score, topic_id)
bgneal@522 162 pipeline.zcard(UPDATED_TOPICS_SET_KEY)
bgneal@522 163 results = pipeline.execute()
bgneal@522 164
bgneal@522 165 # delete topics beyond our maximum count
bgneal@522 166 num_topics = results[-1]
bgneal@522 167 num_to_del = num_topics - MAX_UPDATED_TOPICS
bgneal@522 168 if num_to_del > 0:
bgneal@522 169 # get the IDs of the topics we need to delete first
bgneal@522 170 start = 0
bgneal@522 171 stop = num_to_del - 1 # Redis indices are inclusive
bgneal@523 172 old_ids = conn.zrange(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 173
bgneal@522 174 keys = [UPDATED_TOPIC_KEY % n for n in old_ids]
bgneal@523 175 conn.delete(*keys)
bgneal@522 176
bgneal@522 177 # now delete the oldest num_to_del topics
bgneal@523 178 conn.zremrangebyrank(UPDATED_TOPICS_SET_KEY, start, stop)
bgneal@522 179
bgneal@522 180
bgneal@509 181 def get_latest_posts(num_posts=MAX_POSTS, forum_id=None):
bgneal@509 182 """
bgneal@509 183 This function retrieves num_posts latest posts for the forum with the given
bgneal@509 184 forum_id. If forum_id is None, the posts are retrieved from the combined
bgneal@509 185 forums datastore. A list of dictionaries is returned. Each dictionary
bgneal@509 186 contains information about a post.
bgneal@509 187
bgneal@509 188 """
bgneal@509 189 key = 'forums:latest:%d' % forum_id if forum_id else 'forums:latest:*'
bgneal@509 190
bgneal@509 191 num_posts = max(0, min(MAX_POSTS, num_posts))
bgneal@509 192
bgneal@509 193 if num_posts == 0:
bgneal@509 194 return []
bgneal@509 195
bgneal@523 196 conn = get_redis_connection()
bgneal@523 197 raw_posts = conn.lrange(key, 0, num_posts - 1)
bgneal@509 198
bgneal@509 199 posts = []
bgneal@509 200 for raw_post in raw_posts:
bgneal@509 201 post = simplejson.loads(raw_post)
bgneal@509 202
bgneal@509 203 # fix up the pubdate; turn it back into a datetime object
bgneal@509 204 post['pubdate'] = datetime.datetime.fromtimestamp(post['pubdate'])
bgneal@509 205
bgneal@509 206 posts.append(post)
bgneal@509 207
bgneal@509 208 return posts
bgneal@522 209
bgneal@522 210
bgneal@522 211 @receiver(topic_content_update, dispatch_uid='forums.latest_posts')
bgneal@522 212 def on_topic_update(sender, **kwargs):
bgneal@522 213 """
bgneal@522 214 This function is our signal handler, called when a topic has been updated.
bgneal@522 215 We only care about newly created topics, and ignore updates.
bgneal@522 216
bgneal@522 217 We kick off a Celery task to perform work outside of the request/response
bgneal@522 218 cycle.
bgneal@522 219
bgneal@522 220 """
bgneal@522 221 # ignore non-new topics
bgneal@522 222 if not kwargs['created']:
bgneal@522 223 return
bgneal@522 224
bgneal@522 225 # Kick off a Celery task to process this new post
bgneal@522 226 forums.tasks.new_topic_task.delay(sender.id)
bgneal@522 227
bgneal@522 228
bgneal@522 229 def process_new_topic(topic_id):
bgneal@522 230 """
bgneal@522 231 This function contains new topic processing. Currently we only update the
bgneal@522 232 topic count statistic.
bgneal@522 233
bgneal@522 234 """
bgneal@522 235 try:
bgneal@522 236 topic = Topic.objects.select_related().get(pk=topic_id)
bgneal@522 237 except Topic.DoesNotExist:
bgneal@522 238 logger.warning("process_new_topic: topic %d does not exist", topic_id)
bgneal@522 239 return
bgneal@522 240
bgneal@522 241 # selectively process topics from non-public forums
bgneal@522 242 public_forums = Forum.objects.public_forum_ids()
bgneal@522 243
bgneal@522 244 if topic.forum.id not in public_forums:
bgneal@522 245 return
bgneal@522 246
bgneal@522 247 # update the topic count statistic
bgneal@523 248 conn = get_redis_connection()
bgneal@522 249
bgneal@523 250 result = conn.incr(TOPIC_COUNT_KEY)
bgneal@522 251 if result == 1:
bgneal@522 252 # it is likely redis got trashed, so re-compute the correct value
bgneal@522 253
bgneal@522 254 count = Topic.objects.filter(forum__in=public_forums).count()
bgneal@523 255 conn.set(TOPIC_COUNT_KEY, count)
bgneal@522 256
bgneal@522 257
bgneal@522 258 def get_stats():
bgneal@522 259 """
bgneal@522 260 This function returns the topic and post count statistics as a tuple, in
bgneal@522 261 that order. If a statistic is not available, its position in the tuple will
bgneal@522 262 be None.
bgneal@522 263
bgneal@522 264 """
bgneal@522 265 try:
bgneal@523 266 conn = get_redis_connection()
bgneal@523 267 result = conn.mget(TOPIC_COUNT_KEY, POST_COUNT_KEY)
bgneal@522 268 except redis.RedisError, e:
bgneal@522 269 logger.error(e)
bgneal@522 270 return (None, None)
bgneal@522 271
bgneal@522 272 topic_count = int(result[0]) if result[0] else None
bgneal@522 273 post_count = int(result[1]) if result[1] else None
bgneal@522 274
bgneal@522 275 return (topic_count, post_count)
bgneal@522 276
bgneal@522 277
bgneal@522 278 def get_latest_topic_ids(num):
bgneal@522 279 """
bgneal@522 280 Return a list of topic ids from the latest topics that have posts. The ids
bgneal@522 281 will be sorted from newest to oldest.
bgneal@522 282
bgneal@522 283 """
bgneal@522 284 try:
bgneal@523 285 conn = get_redis_connection()
bgneal@523 286 result = conn.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 287 except redis.RedisError, e:
bgneal@522 288 logger.error(e)
bgneal@522 289 return []
bgneal@522 290
bgneal@522 291 return [int(n) for n in result]
bgneal@522 292
bgneal@522 293
bgneal@522 294 def get_latest_topics(num):
bgneal@522 295 """
bgneal@522 296 Return a list of dictionaries with information about the latest topics that
bgneal@522 297 have updated posts. The topics are sorted from newest to oldest.
bgneal@522 298
bgneal@522 299 """
bgneal@522 300 try:
bgneal@523 301 conn = get_redis_connection()
bgneal@523 302 result = conn.zrevrange(UPDATED_TOPICS_SET_KEY, 0, num - 1)
bgneal@522 303
bgneal@522 304 topic_keys = [UPDATED_TOPIC_KEY % n for n in result]
bgneal@524 305 json_list = conn.mget(topic_keys) if topic_keys else []
bgneal@522 306
bgneal@522 307 except redis.RedisError, e:
bgneal@522 308 logger.error(e)
bgneal@522 309 return []
bgneal@522 310
bgneal@522 311 topics = []
bgneal@522 312 for s in json_list:
bgneal@522 313 item = simplejson.loads(s)
bgneal@522 314 item['date'] = datetime.datetime.fromtimestamp(item['date'])
bgneal@522 315 topics.append(item)
bgneal@522 316
bgneal@522 317 return topics
bgneal@522 318
bgneal@522 319
bgneal@522 320 def notify_topic_delete(topic):
bgneal@522 321 """
bgneal@522 322 This function should be called when a topic is deleted. It will remove the
bgneal@522 323 topic from the updated topics set, if present, and delete any info we have
bgneal@522 324 about the topic.
bgneal@522 325
bgneal@522 326 Note we don't do anything like this for posts. Since they just populate RSS
bgneal@522 327 feeds we'll let them 404. The updated topic list is seen in a prominent
bgneal@522 328 template tag however, so it is a bit more important to get that cleaned up.
bgneal@522 329
bgneal@522 330 """
bgneal@522 331 try:
bgneal@523 332 conn = get_redis_connection()
bgneal@523 333 pipeline = conn.pipeline()
bgneal@522 334 pipeline.zrem(UPDATED_TOPICS_SET_KEY, topic.id)
bgneal@522 335 pipeline.delete(UPDATED_TOPIC_KEY % topic.id)
bgneal@522 336 pipeline.execute()
bgneal@522 337 except redis.RedisError, e:
bgneal@522 338 logger.error(e)
bgneal@522 339
bgneal@522 340
bgneal@522 341 # Down here to avoid a circular import
bgneal@522 342 import forums.tasks