# HG changeset patch # User Brian Neal # Date 1324150164 0 # Node ID f72ace06658a9fa1b21f667fdfd2ec7b06679b35 # Parent 5171a5e9353b3fec53360b2521e9729919dc605b For #194, rework the who's online and max users functions. diff -r 5171a5e9353b -r f72ace06658a gpp/core/management/commands/max_users.py --- a/gpp/core/management/commands/max_users.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/core/management/commands/max_users.py Sat Dec 17 19:29:24 2011 +0000 @@ -7,41 +7,11 @@ from django.core.management.base import NoArgsCommand -from core.models import Statistic -from core.whos_online import get_users_online, get_visitors_online, tick +from core.whos_online import max_users class Command(NoArgsCommand): help = "Run periodically to compute the max users online statistic." def handle_noargs(self, **options): - - now = datetime.datetime.now() - - users = len(get_users_online()) - guests = len(get_visitors_online()) - - updated = False - try: - stat = Statistic.objects.get(pk=1) - except Statistic.DoesNotExist: - stat = Statistic(max_users=users, - max_users_date=now, - max_anon_users=guests, - max_anon_users_date=now) - updated=True - else: - if users > stat.max_users: - stat.max_users = users - stat.max_users_date = now - updated=True - if guests > stat.max_anon_users: - stat.max_anon_users = guests - stat.max_anon_users_date = now - updated=True - - if updated: - stat.save() - - # "tick" the who's online data collector - tick() + max_users() diff -r 5171a5e9353b -r f72ace06658a gpp/core/services.py --- a/gpp/core/services.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/core/services.py Sat Dec 17 19:29:24 2011 +0000 @@ -18,4 +18,4 @@ Create and return a Redis connection using the supplied parameters. """ - return redis.Redis(host=host, port=port, db=db) + return redis.StrictRedis(host=host, port=port, db=db) diff -r 5171a5e9353b -r f72ace06658a gpp/core/tasks.py --- a/gpp/core/tasks.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/core/tasks.py Sat Dec 17 19:29:24 2011 +0000 @@ -5,6 +5,8 @@ from celery.task import task import django.core.mail +import core.whos_online + @task def add(x, y): @@ -47,3 +49,13 @@ command = ForumCleanup() command.execute() + + +@task +def max_users(): + """ + Run the periodic task to calculate the who's online max users/visitors + statistics. + + """ + core.whos_online.max_users() diff -r 5171a5e9353b -r f72ace06658a gpp/core/templatetags/core_tags.py --- a/gpp/core/templatetags/core_tags.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/core/templatetags/core_tags.py Sat Dec 17 19:29:24 2011 +0000 @@ -1,5 +1,6 @@ """ Miscellaneous/utility template tags. + """ import collections import datetime @@ -128,7 +129,7 @@ """ A simple named tuple-type class for birthdays. This class was created to make things easier in the template. - + """ day = None profiles = [] @@ -184,7 +185,7 @@ @register.tag def encode_params(parser, token): """ - This is the compilation function for the encore_params template tag. + This is the compilation function for the encode_params template tag. This template tag retrieves the named parameters from the supplied querydict and returns them as a urlencoded string. diff -r 5171a5e9353b -r f72ace06658a gpp/core/whos_online.py --- a/gpp/core/whos_online.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/core/whos_online.py Sat Dec 17 19:29:24 2011 +0000 @@ -2,27 +2,28 @@ This module keeps track of who is online. We maintain records for both authenticated users ("users") and non-authenticated visitors ("visitors"). """ +import datetime import logging +import time import redis from core.services import get_redis_connection +from core.models import Statistic -# Users and visitors each have 2 sets that we store in a Redis database: -# a current set and an old set. Whenever a user or visitor is seen, the -# current set is updated. At some interval, the current set is renamed -# to the old set, thus destroying it. At any given time, the union of the -# current and old sets is the "who's online" set. +# Users and visitors each have a sorted set in a Redis database. When a user or +# visitor is seen, the respective set is updated with the score of the current +# time. Periodically we remove elements by score (time) to stale out members. # Redis key names: -USER_CURRENT_KEY = "wo_user_current" -USER_OLD_KEY = "wo_user_old" -USER_KEYS = [USER_CURRENT_KEY, USER_OLD_KEY] +USER_SET_KEY = "whos_online:users" +VISITOR_SET_KEY = "whos_online:visitors" -VISITOR_CURRENT_KEY = "wo_visitor_current" -VISITOR_OLD_KEY = "wo_visitor_old" -VISITOR_KEYS = [VISITOR_CURRENT_KEY, VISITOR_OLD_KEY] +CORE_STATS_KEY = "core:stats" + +# the period over which we collect who's online stats: +MAX_AGE = datetime.timedelta(minutes=15) # Logging: we don't want a Redis malfunction to bring down the site. So we @@ -43,87 +44,196 @@ return None +def to_timestamp(dt): + """ + Turn the supplied datetime object into a UNIX timestamp integer. + + """ + return int(time.mktime(dt.timetuple())) + + +def _zadd(key, member): + """ + Adds the member to the given set key, using the current time as the score. + + """ + conn = _get_connection() + if conn: + ts = to_timestamp(datetime.datetime.now()) + try: + conn.zadd(key, ts, member) + except redis.RedisError, e: + logger.error(e) + + +def _zrangebyscore(key): + """ + Performs a zrangebyscore operation on the set given by key. + The minimum score will be a timestap equal to the current time + minus MAX_AGE. The maximum score will be a timestap equal to the + current time. + + """ + conn = _get_connection() + if conn: + now = datetime.datetime.now() + min = to_timestamp(now - MAX_AGE) + max = to_timestamp(now) + try: + return conn.zrangebyscore(key, min, max) + except redis.RedisError, e: + logger.error(e) + + return [] + + def report_user(username): """ Call this function when a user has been seen. The username will be added to - the current set. + the set of users online. + """ - conn = _get_connection() - if conn: - try: - conn.sadd(USER_CURRENT_KEY, username) - except redis.RedisError, e: - logger.error(e) + _zadd(USER_SET_KEY, username) def report_visitor(ip): """ Call this function when a visitor has been seen. The IP address will be - added the current set. + added to the set of visitors online. + + """ + _zadd(VISITOR_SET_KEY, ip) + + +def get_users_online(): + """ + Returns a list of user names from the user set. + sets. + """ + return _zrangebyscore(USER_SET_KEY) + + +def get_visitors_online(): + """ + Returns a list of visitor IP addresses from the visitor set. + """ + return _zrangebyscore(VISITOR_SET_KEY) + + +def _tick(conn): + """ + Call this function to "age out" the sets by removing old users/visitors. + It then returns a tuple of the form: + (zcard users, zcard visitors) + + """ + cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE) + + try: + pipeline = conn.pipeline(transaction=False) + pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff) + pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff) + pipeline.zcard(USER_SET_KEY) + pipeline.zcard(VISITOR_SET_KEY) + result = pipeline.execute() + except redis.RedisError, e: + logger.error(e) + return 0, 0 + + return result[2], result[3] + + +def max_users(): + """ + Run this function periodically to clean out the sets and to compute our max + users and max visitors statistics. + """ conn = _get_connection() + if not conn: + return + + num_users, num_visitors = _tick(conn) + now = datetime.datetime.now() + + stats = get_stats(conn) + update = False + + if stats is None: + stats = Statistic(id=1, + max_users=num_users, + max_users_date=now, + max_anon_users=num_visitors, + max_anon_users_date=now) + update = True + else: + if num_users > stats.max_users: + stats.max_users = num_users + stats.max_users_date = now + update = True + + if num_visitors > stats.max_anon_users: + stats.max_anon_users = num_visitors + stats.max_anon_users_date = now + update = True + + if update: + _save_stats_to_redis(conn, stats) + stats.save() + + +def get_stats(conn=None): + """ + This function retrieves the who's online max user stats out of Redis. If + the keys do not exist in Redis, we fall back to the database. If the stats + are not available, None is returned. + Note that if we can find stats data, it will be returned as a Statistic + object. + + """ + if conn is None: + conn = _get_connection() + + stats = None if conn: try: - conn.sadd(VISITOR_CURRENT_KEY, ip) + stats = conn.hgetall(CORE_STATS_KEY) except redis.RedisError, e: logger.error(e) + if stats: + return Statistic( + id=1, + max_users=stats['max_users'], + max_users_date=datetime.datetime.fromtimestamp( + float(stats['max_users_date'])), + max_anon_users=stats['max_anon_users'], + max_anon_users_date=datetime.datetime.fromtimestamp( + float(stats['max_anon_users_date']))) -def get_users_online(): + try: + stats = Statistic.objects.get(pk=1) + except Statistic.DoesNotExist: + return None + else: + _save_stats_to_redis(conn, stats) + return stats + + +def _save_stats_to_redis(conn, stats): """ - Returns a set of user names which is the union of the current and old - sets. + Saves the statistics to Redis. A TTL is put on the key to prevent Redis and + the database from becoming out of sync. + """ - conn = _get_connection() - if conn: - try: - # Note that keys that do not exist are considered empty sets - return conn.sunion(USER_KEYS) - except redis.RedisError, e: - logger.error(e) + fields = dict( + max_users=stats.max_users, + max_users_date=to_timestamp(stats.max_users_date), + max_anon_users=stats.max_anon_users, + max_anon_users_date=to_timestamp(stats.max_anon_users_date)) - return set() - - -def get_visitors_online(): - """ - Returns a set of visitor IP addresses which is the union of the current - and old sets. - """ - conn = _get_connection() - if conn: - try: - # Note that keys that do not exist are considered empty sets - return conn.sunion(VISITOR_KEYS) - except redis.RedisError, e: - logger.error(e) - - return set() - - -def _tick_set(conn, current, old): - """ - This function renames the set named "current" to "old". - """ - # An exception may be raised if the current key doesn't exist; if that - # happens we have to delete the old set because no one is online. try: - conn.rename(current, old) - except redis.ResponseError: - try: - del conn[old] - except redis.RedisError, e: - logger.error(e) + conn.hmset(CORE_STATS_KEY, fields) + conn.expire(CORE_STATS_KEY, 4 * 60 * 60) except redis.RedisError, e: logger.error(e) - - -def tick(): - """ - Call this function to "age out" the old sets by renaming the current sets - to the old. - """ - conn = _get_connection() - if conn: - _tick_set(conn, USER_CURRENT_KEY, USER_OLD_KEY) - _tick_set(conn, VISITOR_CURRENT_KEY, VISITOR_OLD_KEY) diff -r 5171a5e9353b -r f72ace06658a gpp/forums/templatetags/forum_tags.py --- a/gpp/forums/templatetags/forum_tags.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/forums/templatetags/forum_tags.py Sat Dec 17 19:29:24 2011 +0000 @@ -14,7 +14,7 @@ from forums.models import Post from forums.models import Category from forums.stats import retrieve_stats -from core.models import Statistic +from core.whos_online import get_stats as get_core_stats register = template.Library() @@ -154,15 +154,10 @@ """ Displays forum statistics. """ - try: - stats = Statistic.objects.get(pk=1) - except Statistic.DoesNotExist: - stats = None - post_count, user_count, latest_user = retrieve_stats() return { - 'stats': stats, + 'stats': get_core_stats(), 'post_count': post_count, 'user_count': user_count, 'latest_user': latest_user, diff -r 5171a5e9353b -r f72ace06658a gpp/settings/base.py --- a/gpp/settings/base.py Fri Dec 16 01:17:35 2011 +0000 +++ b/gpp/settings/base.py Sat Dec 17 19:29:24 2011 +0000 @@ -224,10 +224,14 @@ "task": "core.tasks.cleanup", "schedule": crontab(minute=0, hour=1), }, - "purge messages": { + "purge_messages": { "task": "messages.tasks.purge_messages", "schedule": crontab(minute=30, hour=1, day_of_week='sunday'), - } + }, + "max_users": { + "task": "core.tasks.max_users", + "schedule": crontab(minute='*/15'), + }, } #######################################################################