bgneal@423: """
bgneal@423: This module keeps track of who is online. We maintain records for both
bgneal@423: authenticated users ("users") and non-authenticated visitors ("visitors").
bgneal@423: """
bgneal@519: import datetime
bgneal@423: import logging
bgneal@519: import time
bgneal@423:
bgneal@423: import redis
bgneal@423:
bgneal@508: from core.services import get_redis_connection
bgneal@519: from core.models import Statistic
bgneal@508:
bgneal@508:
bgneal@519: # Users and visitors each have a sorted set in a Redis database. When a user or
bgneal@519: # visitor is seen, the respective set is updated with the score of the current
bgneal@519: # time. Periodically we remove elements by score (time) to stale out members.
bgneal@423:
bgneal@423: # Redis key names:
bgneal@519: USER_SET_KEY = "whos_online:users"
bgneal@519: VISITOR_SET_KEY = "whos_online:visitors"
bgneal@423:
bgneal@519: CORE_STATS_KEY = "core:stats"
bgneal@519:
bgneal@519: # the period over which we collect who's online stats:
bgneal@519: MAX_AGE = datetime.timedelta(minutes=15)
bgneal@423:
bgneal@423:
bgneal@423: # Logging: we don't want a Redis malfunction to bring down the site. So we
bgneal@423: # catch all Redis exceptions, log them, and press on.
bgneal@423: logger = logging.getLogger(__name__)
bgneal@423:
bgneal@423:
bgneal@423: def _get_connection():
bgneal@423: """
bgneal@423: Create and return a Redis connection. Returns None on failure.
bgneal@423: """
bgneal@423: try:
bgneal@508: conn = get_redis_connection()
bgneal@423: return conn
bgneal@423: except redis.RedisError, e:
bgneal@423: logger.error(e)
bgneal@423:
bgneal@423: return None
bgneal@423:
bgneal@423:
bgneal@519: def to_timestamp(dt):
bgneal@519: """
bgneal@519: Turn the supplied datetime object into a UNIX timestamp integer.
bgneal@519:
bgneal@519: """
bgneal@519: return int(time.mktime(dt.timetuple()))
bgneal@519:
bgneal@519:
bgneal@519: def _zadd(key, member):
bgneal@519: """
bgneal@519: Adds the member to the given set key, using the current time as the score.
bgneal@519:
bgneal@519: """
bgneal@519: conn = _get_connection()
bgneal@519: if conn:
bgneal@519: ts = to_timestamp(datetime.datetime.now())
bgneal@519: try:
bgneal@519: conn.zadd(key, ts, member)
bgneal@519: except redis.RedisError, e:
bgneal@519: logger.error(e)
bgneal@519:
bgneal@519:
bgneal@519: def _zrangebyscore(key):
bgneal@519: """
bgneal@519: Performs a zrangebyscore operation on the set given by key.
bgneal@519: The minimum score will be a timestap equal to the current time
bgneal@519: minus MAX_AGE. The maximum score will be a timestap equal to the
bgneal@519: current time.
bgneal@519:
bgneal@519: """
bgneal@519: conn = _get_connection()
bgneal@519: if conn:
bgneal@519: now = datetime.datetime.now()
bgneal@519: min = to_timestamp(now - MAX_AGE)
bgneal@519: max = to_timestamp(now)
bgneal@519: try:
bgneal@519: return conn.zrangebyscore(key, min, max)
bgneal@519: except redis.RedisError, e:
bgneal@519: logger.error(e)
bgneal@519:
bgneal@519: return []
bgneal@519:
bgneal@519:
bgneal@423: def report_user(username):
bgneal@423: """
bgneal@423: Call this function when a user has been seen. The username will be added to
bgneal@519: the set of users online.
bgneal@519:
bgneal@423: """
bgneal@519: _zadd(USER_SET_KEY, username)
bgneal@423:
bgneal@423:
bgneal@423: def report_visitor(ip):
bgneal@423: """
bgneal@423: Call this function when a visitor has been seen. The IP address will be
bgneal@519: added to the set of visitors online.
bgneal@519:
bgneal@519: """
bgneal@519: _zadd(VISITOR_SET_KEY, ip)
bgneal@519:
bgneal@519:
bgneal@519: def get_users_online():
bgneal@519: """
bgneal@519: Returns a list of user names from the user set.
bgneal@519: sets.
bgneal@519: """
bgneal@519: return _zrangebyscore(USER_SET_KEY)
bgneal@519:
bgneal@519:
bgneal@519: def get_visitors_online():
bgneal@519: """
bgneal@519: Returns a list of visitor IP addresses from the visitor set.
bgneal@519: """
bgneal@519: return _zrangebyscore(VISITOR_SET_KEY)
bgneal@519:
bgneal@519:
bgneal@519: def _tick(conn):
bgneal@519: """
bgneal@519: Call this function to "age out" the sets by removing old users/visitors.
bgneal@519: It then returns a tuple of the form:
bgneal@519: (zcard users, zcard visitors)
bgneal@519:
bgneal@519: """
bgneal@519: cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
bgneal@519:
bgneal@519: try:
bgneal@519: pipeline = conn.pipeline(transaction=False)
bgneal@519: pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
bgneal@519: pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
bgneal@519: pipeline.zcard(USER_SET_KEY)
bgneal@519: pipeline.zcard(VISITOR_SET_KEY)
bgneal@519: result = pipeline.execute()
bgneal@519: except redis.RedisError, e:
bgneal@519: logger.error(e)
bgneal@519: return 0, 0
bgneal@519:
bgneal@519: return result[2], result[3]
bgneal@519:
bgneal@519:
bgneal@519: def max_users():
bgneal@519: """
bgneal@519: Run this function periodically to clean out the sets and to compute our max
bgneal@519: users and max visitors statistics.
bgneal@519:
bgneal@423: """
bgneal@423: conn = _get_connection()
bgneal@519: if not conn:
bgneal@519: return
bgneal@519:
bgneal@519: num_users, num_visitors = _tick(conn)
bgneal@519: now = datetime.datetime.now()
bgneal@519:
bgneal@519: stats = get_stats(conn)
bgneal@519: update = False
bgneal@519:
bgneal@519: if stats is None:
bgneal@519: stats = Statistic(id=1,
bgneal@519: max_users=num_users,
bgneal@519: max_users_date=now,
bgneal@519: max_anon_users=num_visitors,
bgneal@519: max_anon_users_date=now)
bgneal@519: update = True
bgneal@519: else:
bgneal@519: if num_users > stats.max_users:
bgneal@519: stats.max_users = num_users
bgneal@519: stats.max_users_date = now
bgneal@519: update = True
bgneal@519:
bgneal@519: if num_visitors > stats.max_anon_users:
bgneal@519: stats.max_anon_users = num_visitors
bgneal@519: stats.max_anon_users_date = now
bgneal@519: update = True
bgneal@519:
bgneal@519: if update:
bgneal@519: _save_stats_to_redis(conn, stats)
bgneal@519: stats.save()
bgneal@519:
bgneal@519:
bgneal@519: def get_stats(conn=None):
bgneal@519: """
bgneal@519: This function retrieves the who's online max user stats out of Redis. If
bgneal@519: the keys do not exist in Redis, we fall back to the database. If the stats
bgneal@519: are not available, None is returned.
bgneal@519: Note that if we can find stats data, it will be returned as a Statistic
bgneal@519: object.
bgneal@519:
bgneal@519: """
bgneal@519: if conn is None:
bgneal@519: conn = _get_connection()
bgneal@519:
bgneal@519: stats = None
bgneal@423: if conn:
bgneal@423: try:
bgneal@519: stats = conn.hgetall(CORE_STATS_KEY)
bgneal@423: except redis.RedisError, e:
bgneal@423: logger.error(e)
bgneal@423:
bgneal@519: if stats:
bgneal@519: return Statistic(
bgneal@519: id=1,
bgneal@599: max_users=int(stats['max_users']),
bgneal@519: max_users_date=datetime.datetime.fromtimestamp(
bgneal@519: float(stats['max_users_date'])),
bgneal@599: max_anon_users=int(stats['max_anon_users']),
bgneal@519: max_anon_users_date=datetime.datetime.fromtimestamp(
bgneal@519: float(stats['max_anon_users_date'])))
bgneal@423:
bgneal@519: try:
bgneal@519: stats = Statistic.objects.get(pk=1)
bgneal@519: except Statistic.DoesNotExist:
bgneal@519: return None
bgneal@519: else:
bgneal@519: _save_stats_to_redis(conn, stats)
bgneal@519: return stats
bgneal@519:
bgneal@519:
bgneal@519: def _save_stats_to_redis(conn, stats):
bgneal@423: """
bgneal@519: Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
bgneal@519: the database from becoming out of sync.
bgneal@519:
bgneal@423: """
bgneal@519: fields = dict(
bgneal@519: max_users=stats.max_users,
bgneal@519: max_users_date=to_timestamp(stats.max_users_date),
bgneal@519: max_anon_users=stats.max_anon_users,
bgneal@519: max_anon_users_date=to_timestamp(stats.max_anon_users_date))
bgneal@423:
bgneal@423: try:
bgneal@519: conn.hmset(CORE_STATS_KEY, fields)
bgneal@519: conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
bgneal@423: except redis.RedisError, e:
bgneal@423: logger.error(e)