--- a/gpp/core/management/commands/max_users.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/core/management/commands/max_users.py	Sat Dec 17 19:29:24 2011 +0000
@@ -7,41 +7,11 @@
 
 from django.core.management.base import NoArgsCommand
 
-from core.models import Statistic
-from core.whos_online import get_users_online, get_visitors_online, tick
+from core.whos_online import max_users
 
 
 class Command(NoArgsCommand):
     help = "Run periodically to compute the max users online statistic."
 
     def handle_noargs(self, **options):
-
-        now = datetime.datetime.now()
-
-        users = len(get_users_online())
-        guests = len(get_visitors_online())
-
-        updated = False
-        try:
-            stat = Statistic.objects.get(pk=1)
-        except Statistic.DoesNotExist:
-            stat = Statistic(max_users=users,
-                    max_users_date=now,
-                    max_anon_users=guests,
-                    max_anon_users_date=now)
-            updated=True
-        else:
-            if users > stat.max_users:
-                stat.max_users = users
-                stat.max_users_date = now
-                updated=True
-            if guests > stat.max_anon_users:
-                stat.max_anon_users = guests
-                stat.max_anon_users_date = now
-                updated=True
-
-        if updated:
-            stat.save()
-
-        # "tick" the who's online data collector
-        tick()
+        max_users()

--- a/gpp/core/services.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/core/services.py	Sat Dec 17 19:29:24 2011 +0000
@@ -18,4 +18,4 @@
     Create and return a Redis connection using the supplied parameters.
 
     """
-    return redis.Redis(host=host, port=port, db=db)
+    return redis.StrictRedis(host=host, port=port, db=db)

--- a/gpp/core/tasks.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/core/tasks.py	Sat Dec 17 19:29:24 2011 +0000
@@ -5,6 +5,8 @@
 from celery.task import task
 import django.core.mail
 
+import core.whos_online
+
 
 @task
 def add(x, y):
@@ -47,3 +49,13 @@
 
     command = ForumCleanup()
     command.execute()
+
+
+@task
+def max_users():
+    """
+    Run the periodic task to calculate the who's online max users/visitors
+    statistics.
+
+    """
+    core.whos_online.max_users()

--- a/gpp/core/templatetags/core_tags.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/core/templatetags/core_tags.py	Sat Dec 17 19:29:24 2011 +0000
@@ -1,5 +1,6 @@
 """
 Miscellaneous/utility template tags.
+
 """
 import collections
 import datetime
@@ -128,7 +129,7 @@
     """
     A simple named tuple-type class for birthdays.
     This class was created to make things easier in the template.
-    
+
     """
     day = None
     profiles = []
@@ -184,7 +185,7 @@
 @register.tag
 def encode_params(parser, token):
     """
-    This is the compilation function for the encore_params template tag.
+    This is the compilation function for the encode_params template tag.
     This template tag retrieves the named parameters from the supplied
     querydict and returns them as a urlencoded string.
 

--- a/gpp/core/whos_online.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/core/whos_online.py	Sat Dec 17 19:29:24 2011 +0000
@@ -2,27 +2,28 @@
 This module keeps track of who is online. We maintain records for both
 authenticated users ("users") and non-authenticated visitors ("visitors").
 """
+import datetime
 import logging
+import time
 
 import redis
 
 from core.services import get_redis_connection
+from core.models import Statistic
 
 
-# Users and visitors each have 2 sets that we store in a Redis database:
-# a current set and an old set. Whenever a user or visitor is seen, the
-# current set is updated. At some interval, the current set is renamed
-# to the old set, thus destroying it. At any given time, the union of the
-# current and old sets is the "who's online" set.
+# Users and visitors each have a sorted set in a Redis database. When a user or
+# visitor is seen, the respective set is updated with the score of the current
+# time. Periodically we remove elements by score (time) to stale out members.
 
 # Redis key names:
-USER_CURRENT_KEY = "wo_user_current"
-USER_OLD_KEY = "wo_user_old"
-USER_KEYS = [USER_CURRENT_KEY, USER_OLD_KEY]
+USER_SET_KEY = "whos_online:users"
+VISITOR_SET_KEY = "whos_online:visitors"
 
-VISITOR_CURRENT_KEY = "wo_visitor_current"
-VISITOR_OLD_KEY = "wo_visitor_old"
-VISITOR_KEYS = [VISITOR_CURRENT_KEY, VISITOR_OLD_KEY]
+CORE_STATS_KEY = "core:stats"
+
+# the period over which we collect who's online stats:
+MAX_AGE = datetime.timedelta(minutes=15)
 
 
 # Logging: we don't want a Redis malfunction to bring down the site. So we
@@ -43,87 +44,196 @@
     return None
 
 
+def to_timestamp(dt):
+    """
+    Turn the supplied datetime object into a UNIX timestamp integer.
+
+    """
+    return int(time.mktime(dt.timetuple()))
+
+
+def _zadd(key, member):
+    """
+    Adds the member to the given set key, using the current time as the score.
+
+    """
+    conn = _get_connection()
+    if conn:
+        ts = to_timestamp(datetime.datetime.now())
+        try:
+            conn.zadd(key, ts, member)
+        except redis.RedisError, e:
+            logger.error(e)
+
+
+def _zrangebyscore(key):
+    """
+    Performs a zrangebyscore operation on the set given by key.
+    The minimum score will be a timestap equal to the current time
+    minus MAX_AGE. The maximum score will be a timestap equal to the
+    current time.
+
+    """
+    conn = _get_connection()
+    if conn:
+        now = datetime.datetime.now()
+        min = to_timestamp(now - MAX_AGE)
+        max = to_timestamp(now)
+        try:
+            return conn.zrangebyscore(key, min, max)
+        except redis.RedisError, e:
+            logger.error(e)
+
+    return []
+
+
 def report_user(username):
     """
     Call this function when a user has been seen. The username will be added to
-    the current set.
+    the set of users online.
+
     """
-    conn = _get_connection()
-    if conn:
-        try:
-            conn.sadd(USER_CURRENT_KEY, username)
-        except redis.RedisError, e:
-            logger.error(e)
+    _zadd(USER_SET_KEY, username)
 
 
 def report_visitor(ip):
     """
     Call this function when a visitor has been seen. The IP address will be
-    added the current set.
+    added to the set of visitors online.
+
+    """
+    _zadd(VISITOR_SET_KEY, ip)
+
+
+def get_users_online():
+    """
+    Returns a list of user names from the user set.
+    sets.
+    """
+    return _zrangebyscore(USER_SET_KEY)
+
+
+def get_visitors_online():
+    """
+    Returns a list of visitor IP addresses from the visitor set.
+    """
+    return _zrangebyscore(VISITOR_SET_KEY)
+
+
+def _tick(conn):
+    """
+    Call this function to "age out" the sets by removing old users/visitors.
+    It then returns a tuple of the form:
+        (zcard users, zcard visitors)
+
+    """
+    cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
+
+    try:
+        pipeline = conn.pipeline(transaction=False)
+        pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
+        pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
+        pipeline.zcard(USER_SET_KEY)
+        pipeline.zcard(VISITOR_SET_KEY)
+        result = pipeline.execute()
+    except redis.RedisError, e:
+        logger.error(e)
+        return 0, 0
+
+    return result[2], result[3]
+
+
+def max_users():
+    """
+    Run this function periodically to clean out the sets and to compute our max
+    users and max visitors statistics.
+
     """
     conn = _get_connection()
+    if not conn:
+        return
+
+    num_users, num_visitors = _tick(conn)
+    now = datetime.datetime.now()
+
+    stats = get_stats(conn)
+    update = False
+
+    if stats is None:
+        stats = Statistic(id=1,
+                max_users=num_users,
+                max_users_date=now,
+                max_anon_users=num_visitors,
+                max_anon_users_date=now)
+        update = True
+    else:
+        if num_users > stats.max_users:
+            stats.max_users = num_users
+            stats.max_users_date = now
+            update = True
+
+        if num_visitors > stats.max_anon_users:
+            stats.max_anon_users = num_visitors
+            stats.max_anon_users_date = now
+            update = True
+
+    if update:
+        _save_stats_to_redis(conn, stats)
+        stats.save()
+
+
+def get_stats(conn=None):
+    """
+    This function retrieves the who's online max user stats out of Redis. If
+    the keys do not exist in Redis, we fall back to the database. If the stats
+    are not available, None is returned.
+    Note that if we can find stats data, it will be returned as a Statistic
+    object.
+
+    """
+    if conn is None:
+        conn = _get_connection()
+
+    stats = None
     if conn:
         try:
-            conn.sadd(VISITOR_CURRENT_KEY, ip)
+            stats = conn.hgetall(CORE_STATS_KEY)
         except redis.RedisError, e:
             logger.error(e)
 
+        if stats:
+            return Statistic(
+                id=1,
+                max_users=stats['max_users'],
+                max_users_date=datetime.datetime.fromtimestamp(
+                    float(stats['max_users_date'])),
+                max_anon_users=stats['max_anon_users'],
+                max_anon_users_date=datetime.datetime.fromtimestamp(
+                    float(stats['max_anon_users_date'])))
 
-def get_users_online():
+    try:
+        stats = Statistic.objects.get(pk=1)
+    except Statistic.DoesNotExist:
+        return None
+    else:
+        _save_stats_to_redis(conn, stats)
+        return stats
+
+
+def _save_stats_to_redis(conn, stats):
     """
-    Returns a set of user names which is the union of the current and old
-    sets.
+    Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
+    the database from becoming out of sync.
+
     """
-    conn = _get_connection()
-    if conn:
-        try:
-            # Note that keys that do not exist are considered empty sets
-            return conn.sunion(USER_KEYS)
-        except redis.RedisError, e:
-            logger.error(e)
+    fields = dict(
+        max_users=stats.max_users,
+        max_users_date=to_timestamp(stats.max_users_date),
+        max_anon_users=stats.max_anon_users,
+        max_anon_users_date=to_timestamp(stats.max_anon_users_date))
 
-    return set()
-
-
-def get_visitors_online():
-    """
-    Returns a set of visitor IP addresses which is the union of the current
-    and old sets.
-    """
-    conn = _get_connection()
-    if conn:
-        try:
-            # Note that keys that do not exist are considered empty sets
-            return conn.sunion(VISITOR_KEYS)
-        except redis.RedisError, e:
-            logger.error(e)
-
-    return set()
-
-
-def _tick_set(conn, current, old):
-    """
-    This function renames the set named "current" to "old".
-    """
-    # An exception may be raised if the current key doesn't exist; if that
-    # happens we have to delete the old set because no one is online.
     try:
-        conn.rename(current, old)
-    except redis.ResponseError:
-        try:
-            del conn[old]
-        except redis.RedisError, e:
-            logger.error(e)
+        conn.hmset(CORE_STATS_KEY, fields)
+        conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
     except redis.RedisError, e:
         logger.error(e)
-
-
-def tick():
-    """
-    Call this function to "age out" the old sets by renaming the current sets
-    to the old.
-    """
-    conn = _get_connection()
-    if conn:
-        _tick_set(conn, USER_CURRENT_KEY, USER_OLD_KEY)
-        _tick_set(conn, VISITOR_CURRENT_KEY, VISITOR_OLD_KEY)

--- a/gpp/forums/templatetags/forum_tags.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/forums/templatetags/forum_tags.py	Sat Dec 17 19:29:24 2011 +0000
@@ -14,7 +14,7 @@
 from forums.models import Post
 from forums.models import Category
 from forums.stats import retrieve_stats
-from core.models import Statistic
+from core.whos_online import get_stats as get_core_stats
 
 
 register = template.Library()
@@ -154,15 +154,10 @@
     """
     Displays forum statistics.
     """
-    try:
-        stats = Statistic.objects.get(pk=1)
-    except Statistic.DoesNotExist:
-        stats = None
-
     post_count, user_count, latest_user = retrieve_stats()
 
     return {
-        'stats': stats,
+        'stats': get_core_stats(),
         'post_count': post_count,
         'user_count': user_count,
         'latest_user': latest_user,

--- a/gpp/settings/base.py	Fri Dec 16 01:17:35 2011 +0000
+++ b/gpp/settings/base.py	Sat Dec 17 19:29:24 2011 +0000
@@ -224,10 +224,14 @@
         "task": "core.tasks.cleanup",
         "schedule": crontab(minute=0, hour=1),
     },
-    "purge messages": {
+    "purge_messages": {
         "task": "messages.tasks.purge_messages",
         "schedule": crontab(minute=30, hour=1, day_of_week='sunday'),
-    }
+    },
+    "max_users": {
+        "task": "core.tasks.max_users",
+        "schedule": crontab(minute='*/15'),
+    },
 }
 
 #######################################################################