bgneal@423: """
bgneal@423: This module keeps track of who is online. We maintain records for both
bgneal@423: authenticated users ("users") and non-authenticated visitors ("visitors").
bgneal@423: """
bgneal@519: import datetime
bgneal@423: import logging
bgneal@519: import time
bgneal@423: 
bgneal@423: import redis
bgneal@423: 
bgneal@508: from core.services import get_redis_connection
bgneal@519: from core.models import Statistic
bgneal@508: 
bgneal@508: 
bgneal@519: # Users and visitors each have a sorted set in a Redis database. When a user or
bgneal@519: # visitor is seen, the respective set is updated with the score of the current
bgneal@519: # time. Periodically we remove elements by score (time) to stale out members.
bgneal@423: 
bgneal@423: # Redis key names:
bgneal@519: USER_SET_KEY = "whos_online:users"
bgneal@519: VISITOR_SET_KEY = "whos_online:visitors"
bgneal@423: 
bgneal@519: CORE_STATS_KEY = "core:stats"
bgneal@519: 
bgneal@519: # the period over which we collect who's online stats:
bgneal@519: MAX_AGE = datetime.timedelta(minutes=15)
bgneal@423: 
bgneal@423: 
bgneal@423: # Logging: we don't want a Redis malfunction to bring down the site. So we
bgneal@423: # catch all Redis exceptions, log them, and press on.
bgneal@423: logger = logging.getLogger(__name__)
bgneal@423: 
bgneal@423: 
bgneal@423: def _get_connection():
bgneal@423:     """
bgneal@423:     Create and return a Redis connection. Returns None on failure.
bgneal@423:     """
bgneal@423:     try:
bgneal@508:         conn = get_redis_connection()
bgneal@423:         return conn
bgneal@423:     except redis.RedisError, e:
bgneal@423:         logger.error(e)
bgneal@423: 
bgneal@423:     return None
bgneal@423: 
bgneal@423: 
bgneal@519: def to_timestamp(dt):
bgneal@519:     """
bgneal@519:     Turn the supplied datetime object into a UNIX timestamp integer.
bgneal@519: 
bgneal@519:     """
bgneal@519:     return int(time.mktime(dt.timetuple()))
bgneal@519: 
bgneal@519: 
bgneal@519: def _zadd(key, member):
bgneal@519:     """
bgneal@519:     Adds the member to the given set key, using the current time as the score.
bgneal@519: 
bgneal@519:     """
bgneal@519:     conn = _get_connection()
bgneal@519:     if conn:
bgneal@519:         ts = to_timestamp(datetime.datetime.now())
bgneal@519:         try:
bgneal@519:             conn.zadd(key, ts, member)
bgneal@519:         except redis.RedisError, e:
bgneal@519:             logger.error(e)
bgneal@519: 
bgneal@519: 
bgneal@519: def _zrangebyscore(key):
bgneal@519:     """
bgneal@519:     Performs a zrangebyscore operation on the set given by key.
bgneal@519:     The minimum score will be a timestap equal to the current time
bgneal@519:     minus MAX_AGE. The maximum score will be a timestap equal to the
bgneal@519:     current time.
bgneal@519: 
bgneal@519:     """
bgneal@519:     conn = _get_connection()
bgneal@519:     if conn:
bgneal@519:         now = datetime.datetime.now()
bgneal@519:         min = to_timestamp(now - MAX_AGE)
bgneal@519:         max = to_timestamp(now)
bgneal@519:         try:
bgneal@519:             return conn.zrangebyscore(key, min, max)
bgneal@519:         except redis.RedisError, e:
bgneal@519:             logger.error(e)
bgneal@519: 
bgneal@519:     return []
bgneal@519: 
bgneal@519: 
bgneal@423: def report_user(username):
bgneal@423:     """
bgneal@423:     Call this function when a user has been seen. The username will be added to
bgneal@519:     the set of users online.
bgneal@519: 
bgneal@423:     """
bgneal@519:     _zadd(USER_SET_KEY, username)
bgneal@423: 
bgneal@423: 
bgneal@423: def report_visitor(ip):
bgneal@423:     """
bgneal@423:     Call this function when a visitor has been seen. The IP address will be
bgneal@519:     added to the set of visitors online.
bgneal@519: 
bgneal@519:     """
bgneal@519:     _zadd(VISITOR_SET_KEY, ip)
bgneal@519: 
bgneal@519: 
bgneal@519: def get_users_online():
bgneal@519:     """
bgneal@519:     Returns a list of user names from the user set.
bgneal@519:     sets.
bgneal@519:     """
bgneal@519:     return _zrangebyscore(USER_SET_KEY)
bgneal@519: 
bgneal@519: 
bgneal@519: def get_visitors_online():
bgneal@519:     """
bgneal@519:     Returns a list of visitor IP addresses from the visitor set.
bgneal@519:     """
bgneal@519:     return _zrangebyscore(VISITOR_SET_KEY)
bgneal@519: 
bgneal@519: 
bgneal@519: def _tick(conn):
bgneal@519:     """
bgneal@519:     Call this function to "age out" the sets by removing old users/visitors.
bgneal@519:     It then returns a tuple of the form:
bgneal@519:         (zcard users, zcard visitors)
bgneal@519: 
bgneal@519:     """
bgneal@519:     cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
bgneal@519: 
bgneal@519:     try:
bgneal@519:         pipeline = conn.pipeline(transaction=False)
bgneal@519:         pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
bgneal@519:         pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
bgneal@519:         pipeline.zcard(USER_SET_KEY)
bgneal@519:         pipeline.zcard(VISITOR_SET_KEY)
bgneal@519:         result = pipeline.execute()
bgneal@519:     except redis.RedisError, e:
bgneal@519:         logger.error(e)
bgneal@519:         return 0, 0
bgneal@519: 
bgneal@519:     return result[2], result[3]
bgneal@519: 
bgneal@519: 
bgneal@519: def max_users():
bgneal@519:     """
bgneal@519:     Run this function periodically to clean out the sets and to compute our max
bgneal@519:     users and max visitors statistics.
bgneal@519: 
bgneal@423:     """
bgneal@423:     conn = _get_connection()
bgneal@519:     if not conn:
bgneal@519:         return
bgneal@519: 
bgneal@519:     num_users, num_visitors = _tick(conn)
bgneal@519:     now = datetime.datetime.now()
bgneal@519: 
bgneal@519:     stats = get_stats(conn)
bgneal@519:     update = False
bgneal@519: 
bgneal@519:     if stats is None:
bgneal@519:         stats = Statistic(id=1,
bgneal@519:                 max_users=num_users,
bgneal@519:                 max_users_date=now,
bgneal@519:                 max_anon_users=num_visitors,
bgneal@519:                 max_anon_users_date=now)
bgneal@519:         update = True
bgneal@519:     else:
bgneal@519:         if num_users > stats.max_users:
bgneal@519:             stats.max_users = num_users
bgneal@519:             stats.max_users_date = now
bgneal@519:             update = True
bgneal@519: 
bgneal@519:         if num_visitors > stats.max_anon_users:
bgneal@519:             stats.max_anon_users = num_visitors
bgneal@519:             stats.max_anon_users_date = now
bgneal@519:             update = True
bgneal@519: 
bgneal@519:     if update:
bgneal@519:         _save_stats_to_redis(conn, stats)
bgneal@519:         stats.save()
bgneal@519: 
bgneal@519: 
bgneal@519: def get_stats(conn=None):
bgneal@519:     """
bgneal@519:     This function retrieves the who's online max user stats out of Redis. If
bgneal@519:     the keys do not exist in Redis, we fall back to the database. If the stats
bgneal@519:     are not available, None is returned.
bgneal@519:     Note that if we can find stats data, it will be returned as a Statistic
bgneal@519:     object.
bgneal@519: 
bgneal@519:     """
bgneal@519:     if conn is None:
bgneal@519:         conn = _get_connection()
bgneal@519: 
bgneal@519:     stats = None
bgneal@423:     if conn:
bgneal@423:         try:
bgneal@519:             stats = conn.hgetall(CORE_STATS_KEY)
bgneal@423:         except redis.RedisError, e:
bgneal@423:             logger.error(e)
bgneal@423: 
bgneal@519:         if stats:
bgneal@519:             return Statistic(
bgneal@519:                 id=1,
bgneal@519:                 max_users=stats['max_users'],
bgneal@519:                 max_users_date=datetime.datetime.fromtimestamp(
bgneal@519:                     float(stats['max_users_date'])),
bgneal@519:                 max_anon_users=stats['max_anon_users'],
bgneal@519:                 max_anon_users_date=datetime.datetime.fromtimestamp(
bgneal@519:                     float(stats['max_anon_users_date'])))
bgneal@423: 
bgneal@519:     try:
bgneal@519:         stats = Statistic.objects.get(pk=1)
bgneal@519:     except Statistic.DoesNotExist:
bgneal@519:         return None
bgneal@519:     else:
bgneal@519:         _save_stats_to_redis(conn, stats)
bgneal@519:         return stats
bgneal@519: 
bgneal@519: 
bgneal@519: def _save_stats_to_redis(conn, stats):
bgneal@423:     """
bgneal@519:     Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
bgneal@519:     the database from becoming out of sync.
bgneal@519: 
bgneal@423:     """
bgneal@519:     fields = dict(
bgneal@519:         max_users=stats.max_users,
bgneal@519:         max_users_date=to_timestamp(stats.max_users_date),
bgneal@519:         max_anon_users=stats.max_anon_users,
bgneal@519:         max_anon_users_date=to_timestamp(stats.max_anon_users_date))
bgneal@423: 
bgneal@423:     try:
bgneal@519:         conn.hmset(CORE_STATS_KEY, fields)
bgneal@519:         conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
bgneal@423:     except redis.RedisError, e:
bgneal@423:         logger.error(e)