# HG changeset patch # User Brian Neal # Date 1303586378 0 # Node ID 3fe60148f75cfb8c896b4bcd04ca9c3c9f3af5ed # Parent 6309814cd6f7248959801a00659eee85c8c9d5f8 Fixing #203; use Redis for who's online function. diff -r 6309814cd6f7 -r 3fe60148f75c gpp/core/management/commands/clean_last_visit.py --- a/gpp/core/management/commands/clean_last_visit.py Thu Apr 21 02:23:34 2011 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -""" -clean_last_visit is a custom manage.py command. -It is intended to be called from a cron job to clean out old user and anonymous -last visit records. - -""" -import datetime - -from django.core.management.base import NoArgsCommand - -from core.models import UserLastVisit, AnonLastVisit - -USER_LV_AGE = datetime.timedelta(weeks=4) -ANON_LV_AGE = datetime.timedelta(days=1) - - -class Command(NoArgsCommand): - help = "Run periodically to clean out old last visit records." - - def handle_noargs(self, **options): - - now = datetime.datetime.now() - user_cut_off = now - USER_LV_AGE - anon_cut_off = now - ANON_LV_AGE - - UserLastVisit.objects.filter(last_visit__lte=user_cut_off).delete() - AnonLastVisit.objects.filter(last_visit__lte=anon_cut_off).delete() diff -r 6309814cd6f7 -r 3fe60148f75c gpp/core/management/commands/max_users.py --- a/gpp/core/management/commands/max_users.py Thu Apr 21 02:23:34 2011 +0000 +++ b/gpp/core/management/commands/max_users.py Sat Apr 23 19:19:38 2011 +0000 @@ -7,7 +7,8 @@ from django.core.management.base import NoArgsCommand -from core.models import UserLastVisit, AnonLastVisit, Statistic +from core.models import Statistic +from core.whos_online import get_users_online, get_visitors_online, tick class Command(NoArgsCommand): @@ -16,10 +17,9 @@ def handle_noargs(self, **options): now = datetime.datetime.now() - cut_off = now - datetime.timedelta(minutes=15) - users = UserLastVisit.objects.filter(last_visit__gte=cut_off).count() - guests = AnonLastVisit.objects.filter(last_visit__gte=cut_off).count() + users = len(get_users_online()) + guests = len(get_visitors_online()) updated = False try: @@ -42,3 +42,6 @@ if updated: stat.save() + + # "tick" the who's online data collector + tick() diff -r 6309814cd6f7 -r 3fe60148f75c gpp/core/middleware.py --- a/gpp/core/middleware.py Thu Apr 21 02:23:34 2011 +0000 +++ b/gpp/core/middleware.py Sat Apr 23 19:19:38 2011 +0000 @@ -6,9 +6,8 @@ from django.contrib.auth import logout from django.conf import settings -from core.models import UserLastVisit -from core.models import AnonLastVisit from core.functions import get_ip +from core.whos_online import report_user, report_visitor class InactiveUserMiddleware(object): @@ -26,7 +25,8 @@ ONLINE_COOKIE = 'sg101_online' # online cookie name -ONLINE_TIMEOUT = 10 * 60 # online cookie lifetime in seconds +ONLINE_TIMEOUT = 5 * 60 # online cookie lifetime in seconds + class WhosOnline(object): """ @@ -50,40 +50,23 @@ if request.user.is_authenticated(): if request.COOKIES.get(ONLINE_COOKIE) is None: - # update the last seen timestamp - try: - ulv = UserLastVisit.objects.get(user=request.user) - except UserLastVisit.DoesNotExist: - ulv = UserLastVisit(user=request.user) + # report that we've seen the user + report_user(request.user.username) - ulv.last_visit = datetime.datetime.now() - ulv.save() - - # set a cookie to expire in 10 minutes or so + # set a cookie to expire response.set_cookie(ONLINE_COOKIE, '1', max_age=ONLINE_TIMEOUT) else: if request.COOKIES.get(settings.CSRF_COOKIE_NAME) is not None: # We have a non-authenticated user that has cookies enabled. This # means we can track them. if request.COOKIES.get(ONLINE_COOKIE) is None: - # update the timestamp for this anonymous visitor + # see if we can get the IP address ip = get_ip(request) if ip: - try: - alv = AnonLastVisit.objects.get(ip=ip) - except AnonLastVisit.DoesNotExist: - alv = AnonLastVisit(ip=ip) + # report that we've seen this visitor + report_visitor(ip) - alv.last_visit = datetime.datetime.now() - - # There is a race condition and sometimes another thread - # saves a record before we do; just log this if it happens. - try: - alv.save() - except IntegrityError: - logging.exception('WhosOnline.process_response') - - # set a cookie to expire in 10 minutes or so + # set a cookie to expire response.set_cookie(ONLINE_COOKIE, '1', max_age=ONLINE_TIMEOUT) return response diff -r 6309814cd6f7 -r 3fe60148f75c gpp/core/models.py --- a/gpp/core/models.py Thu Apr 21 02:23:34 2011 +0000 +++ b/gpp/core/models.py Sat Apr 23 19:19:38 2011 +0000 @@ -7,24 +7,6 @@ from django.contrib.auth.models import User -class UserLastVisit(models.Model): - """ - This model represents timestamps indicating a user's last visit to the - site. - """ - user = models.ForeignKey(User, unique=True) - last_visit = models.DateTimeField(db_index=True) - - -class AnonLastVisit(models.Model): - """ - This model represents timestamps for the last visit from non-authenticated - users. - """ - ip = models.CharField(max_length=16, db_index=True, unique=True) - last_visit = models.DateTimeField(db_index=True) - - class Statistic(models.Model): """ This model keeps track of site statistics. Currently, the only statistic @@ -37,6 +19,6 @@ max_anon_users_date = models.DateTimeField() def __unicode__(self): - return u'%d users on %s' % (self.max_users, + return u'%d users on %s' % (self.max_users, self.max_users_date.strftime('%Y-%m-%d %H:%M:%S')) diff -r 6309814cd6f7 -r 3fe60148f75c gpp/core/templatetags/core_tags.py --- a/gpp/core/templatetags/core_tags.py Thu Apr 21 02:23:34 2011 +0000 +++ b/gpp/core/templatetags/core_tags.py Sat Apr 23 19:19:38 2011 +0000 @@ -9,7 +9,7 @@ import repoze.timeago -from core.models import UserLastVisit, AnonLastVisit +from core.whos_online import get_users_online, get_visitors_online from bio.models import UserProfile @@ -37,26 +37,15 @@ """ Displays a list of who is online. """ - info = cache.get('whos_online') - if info: - return info + users = get_users_online() + visitors = get_visitors_online() - now = datetime.datetime.now() - cutoff = now - datetime.timedelta(minutes=10, seconds=5) - - info = {} - users = UserLastVisit.objects.filter( - last_visit__gte=cutoff).values_list('user__username', flat=True) - num_users = len(users) - info['num_users'] = num_users - info['users'] = sorted(users) - - num_guests = AnonLastVisit.objects.filter(last_visit__gte=cutoff).count() - info['num_guests'] = num_guests - info['total'] = num_users + num_guests - - cache.set('whos_online', info, 60) - return info + return { + 'num_users': len(users), + 'users': sorted(users), + 'num_guests': len(visitors), + 'total': len(users) + len(visitors), + } # A somewhat ugly hack until we decide if we should be using UTC time diff -r 6309814cd6f7 -r 3fe60148f75c gpp/core/whos_online.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gpp/core/whos_online.py Sat Apr 23 19:19:38 2011 +0000 @@ -0,0 +1,133 @@ +""" +This module keeps track of who is online. We maintain records for both +authenticated users ("users") and non-authenticated visitors ("visitors"). +""" +import logging + +from django.conf import settings +import redis + +# Users and visitors each have 2 sets that we store in a Redis database: +# a current set and an old set. Whenever a user or visitor is seen, the +# current set is updated. At some interval, the current set is renamed +# to the old set, thus destroying it. At any given time, the union of the +# current and old sets is the "who's online" set. + +# Redis connection and database settings + +HOST = getattr(settings, 'WHOS_ONLINE_REDIS_HOST', 'localhost') +PORT = getattr(settings, 'WHOS_ONLINE_REDIS_PORT', 6379) +DB = getattr(settings, 'WHOS_ONLINE_REDIS_DB', 0) + +# Redis key names: +USER_CURRENT_KEY = "wo_user_current" +USER_OLD_KEY = "wo_user_old" +USER_KEYS = [USER_CURRENT_KEY, USER_OLD_KEY] + +VISITOR_CURRENT_KEY = "wo_visitor_current" +VISITOR_OLD_KEY = "wo_visitor_old" +VISITOR_KEYS = [VISITOR_CURRENT_KEY, VISITOR_OLD_KEY] + + +# Logging: we don't want a Redis malfunction to bring down the site. So we +# catch all Redis exceptions, log them, and press on. +logger = logging.getLogger(__name__) + + +def _get_connection(): + """ + Create and return a Redis connection. Returns None on failure. + """ + try: + conn = redis.Redis(host=HOST, port=PORT, db=DB) + return conn + except redis.RedisError, e: + logger.error(e) + + return None + + +def report_user(username): + """ + Call this function when a user has been seen. The username will be added to + the current set. + """ + conn = _get_connection() + if conn: + try: + conn.sadd(USER_CURRENT_KEY, username) + except redis.RedisError, e: + logger.error(e) + + +def report_visitor(ip): + """ + Call this function when a visitor has been seen. The IP address will be + added the current set. + """ + conn = _get_connection() + if conn: + try: + conn.sadd(VISITOR_CURRENT_KEY, ip) + except redis.RedisError, e: + logger.error(e) + + +def get_users_online(): + """ + Returns a set of user names which is the union of the current and old + sets. + """ + conn = _get_connection() + if conn: + try: + # Note that keys that do not exist are considered empty sets + return conn.sunion(USER_KEYS) + except redis.RedisError, e: + logger.error(e) + + return set() + + +def get_visitors_online(): + """ + Returns a set of visitor IP addresses which is the union of the current + and old sets. + """ + conn = _get_connection() + if conn: + try: + # Note that keys that do not exist are considered empty sets + return conn.sunion(VISITOR_KEYS) + except redis.RedisError, e: + logger.error(e) + + return set() + + +def _tick_set(conn, current, old): + """ + This function renames the set named "current" to "old". + """ + # An exception may be raised if the current key doesn't exist; if that + # happens we have to delete the old set because no one is online. + try: + conn.rename(current, old) + except redis.ResponseError: + try: + del conn[old] + except redis.RedisError, e: + logger.error(e) + except redis.RedisError, e: + logger.error(e) + + +def tick(): + """ + Call this function to "age out" the old sets by renaming the current sets + to the old. + """ + conn = _get_connection() + if conn: + _tick_set(conn, USER_CURRENT_KEY, USER_OLD_KEY) + _tick_set(conn, VISITOR_CURRENT_KEY, VISITOR_OLD_KEY)