Mercurial > public > sg101

annotate core/whos_online.py @ 861:e4f8d87c3d30

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Configure Markdown logger to reduce noise in logs. Markdown is logging at the INFO level whenever it loads an extension. This looks like it has been fixed in master at GitHub. But until then we will explicitly configure the MARKDOWN logger to log at WARNING or higher.
author Brian Neal <bgneal@gmail.com>
date Mon, 01 Dec 2014 18:36:27 -0600
parents 2f5779e9d8f8
children
rev   line source
bgneal@423 1 """
bgneal@423 2 This module keeps track of who is online. We maintain records for both
bgneal@423 3 authenticated users ("users") and non-authenticated visitors ("visitors").
bgneal@423 4 """
bgneal@519 5 import datetime
bgneal@423 6 import logging
bgneal@519 7 import time
bgneal@423 8
bgneal@423 9 import redis
bgneal@423 10
bgneal@508 11 from core.services import get_redis_connection
bgneal@519 12 from core.models import Statistic
bgneal@508 13
bgneal@508 14
bgneal@519 15 # Users and visitors each have a sorted set in a Redis database. When a user or
bgneal@519 16 # visitor is seen, the respective set is updated with the score of the current
bgneal@519 17 # time. Periodically we remove elements by score (time) to stale out members.
bgneal@423 18
bgneal@423 19 # Redis key names:
bgneal@519 20 USER_SET_KEY = "whos_online:users"
bgneal@519 21 VISITOR_SET_KEY = "whos_online:visitors"
bgneal@423 22
bgneal@519 23 CORE_STATS_KEY = "core:stats"
bgneal@519 24
bgneal@519 25 # the period over which we collect who's online stats:
bgneal@519 26 MAX_AGE = datetime.timedelta(minutes=15)
bgneal@423 27
bgneal@423 28
bgneal@423 29 # Logging: we don't want a Redis malfunction to bring down the site. So we
bgneal@423 30 # catch all Redis exceptions, log them, and press on.
bgneal@423 31 logger = logging.getLogger(__name__)
bgneal@423 32
bgneal@423 33
bgneal@423 34 def _get_connection():
bgneal@423 35 """
bgneal@423 36 Create and return a Redis connection. Returns None on failure.
bgneal@423 37 """
bgneal@423 38 try:
bgneal@508 39 conn = get_redis_connection()
bgneal@423 40 return conn
bgneal@423 41 except redis.RedisError, e:
bgneal@423 42 logger.error(e)
bgneal@423 43
bgneal@423 44 return None
bgneal@423 45
bgneal@423 46
bgneal@519 47 def to_timestamp(dt):
bgneal@519 48 """
bgneal@519 49 Turn the supplied datetime object into a UNIX timestamp integer.
bgneal@519 50
bgneal@519 51 """
bgneal@519 52 return int(time.mktime(dt.timetuple()))
bgneal@519 53
bgneal@519 54
bgneal@519 55 def _zadd(key, member):
bgneal@519 56 """
bgneal@519 57 Adds the member to the given set key, using the current time as the score.
bgneal@519 58
bgneal@519 59 """
bgneal@519 60 conn = _get_connection()
bgneal@519 61 if conn:
bgneal@519 62 ts = to_timestamp(datetime.datetime.now())
bgneal@519 63 try:
bgneal@519 64 conn.zadd(key, ts, member)
bgneal@519 65 except redis.RedisError, e:
bgneal@519 66 logger.error(e)
bgneal@519 67
bgneal@519 68
bgneal@519 69 def _zrangebyscore(key):
bgneal@519 70 """
bgneal@519 71 Performs a zrangebyscore operation on the set given by key.
bgneal@519 72 The minimum score will be a timestap equal to the current time
bgneal@519 73 minus MAX_AGE. The maximum score will be a timestap equal to the
bgneal@519 74 current time.
bgneal@519 75
bgneal@519 76 """
bgneal@519 77 conn = _get_connection()
bgneal@519 78 if conn:
bgneal@519 79 now = datetime.datetime.now()
bgneal@519 80 min = to_timestamp(now - MAX_AGE)
bgneal@519 81 max = to_timestamp(now)
bgneal@519 82 try:
bgneal@519 83 return conn.zrangebyscore(key, min, max)
bgneal@519 84 except redis.RedisError, e:
bgneal@519 85 logger.error(e)
bgneal@519 86
bgneal@519 87 return []
bgneal@519 88
bgneal@519 89
bgneal@423 90 def report_user(username):
bgneal@423 91 """
bgneal@423 92 Call this function when a user has been seen. The username will be added to
bgneal@519 93 the set of users online.
bgneal@519 94
bgneal@423 95 """
bgneal@519 96 _zadd(USER_SET_KEY, username)
bgneal@423 97
bgneal@423 98
bgneal@423 99 def report_visitor(ip):
bgneal@423 100 """
bgneal@423 101 Call this function when a visitor has been seen. The IP address will be
bgneal@519 102 added to the set of visitors online.
bgneal@519 103
bgneal@519 104 """
bgneal@519 105 _zadd(VISITOR_SET_KEY, ip)
bgneal@519 106
bgneal@519 107
bgneal@519 108 def get_users_online():
bgneal@519 109 """
bgneal@519 110 Returns a list of user names from the user set.
bgneal@519 111 sets.
bgneal@519 112 """
bgneal@519 113 return _zrangebyscore(USER_SET_KEY)
bgneal@519 114
bgneal@519 115
bgneal@519 116 def get_visitors_online():
bgneal@519 117 """
bgneal@519 118 Returns a list of visitor IP addresses from the visitor set.
bgneal@519 119 """
bgneal@519 120 return _zrangebyscore(VISITOR_SET_KEY)
bgneal@519 121
bgneal@519 122
bgneal@519 123 def _tick(conn):
bgneal@519 124 """
bgneal@519 125 Call this function to "age out" the sets by removing old users/visitors.
bgneal@519 126 It then returns a tuple of the form:
bgneal@519 127 (zcard users, zcard visitors)
bgneal@519 128
bgneal@519 129 """
bgneal@519 130 cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
bgneal@519 131
bgneal@519 132 try:
bgneal@519 133 pipeline = conn.pipeline(transaction=False)
bgneal@519 134 pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
bgneal@519 135 pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
bgneal@519 136 pipeline.zcard(USER_SET_KEY)
bgneal@519 137 pipeline.zcard(VISITOR_SET_KEY)
bgneal@519 138 result = pipeline.execute()
bgneal@519 139 except redis.RedisError, e:
bgneal@519 140 logger.error(e)
bgneal@519 141 return 0, 0
bgneal@519 142
bgneal@519 143 return result[2], result[3]
bgneal@519 144
bgneal@519 145
bgneal@519 146 def max_users():
bgneal@519 147 """
bgneal@519 148 Run this function periodically to clean out the sets and to compute our max
bgneal@519 149 users and max visitors statistics.
bgneal@519 150
bgneal@423 151 """
bgneal@423 152 conn = _get_connection()
bgneal@519 153 if not conn:
bgneal@519 154 return
bgneal@519 155
bgneal@519 156 num_users, num_visitors = _tick(conn)
bgneal@519 157 now = datetime.datetime.now()
bgneal@519 158
bgneal@519 159 stats = get_stats(conn)
bgneal@519 160 update = False
bgneal@519 161
bgneal@519 162 if stats is None:
bgneal@519 163 stats = Statistic(id=1,
bgneal@519 164 max_users=num_users,
bgneal@519 165 max_users_date=now,
bgneal@519 166 max_anon_users=num_visitors,
bgneal@519 167 max_anon_users_date=now)
bgneal@519 168 update = True
bgneal@519 169 else:
bgneal@519 170 if num_users > stats.max_users:
bgneal@519 171 stats.max_users = num_users
bgneal@519 172 stats.max_users_date = now
bgneal@519 173 update = True
bgneal@519 174
bgneal@519 175 if num_visitors > stats.max_anon_users:
bgneal@519 176 stats.max_anon_users = num_visitors
bgneal@519 177 stats.max_anon_users_date = now
bgneal@519 178 update = True
bgneal@519 179
bgneal@519 180 if update:
bgneal@519 181 _save_stats_to_redis(conn, stats)
bgneal@519 182 stats.save()
bgneal@519 183
bgneal@519 184
bgneal@519 185 def get_stats(conn=None):
bgneal@519 186 """
bgneal@519 187 This function retrieves the who's online max user stats out of Redis. If
bgneal@519 188 the keys do not exist in Redis, we fall back to the database. If the stats
bgneal@519 189 are not available, None is returned.
bgneal@519 190 Note that if we can find stats data, it will be returned as a Statistic
bgneal@519 191 object.
bgneal@519 192
bgneal@519 193 """
bgneal@519 194 if conn is None:
bgneal@519 195 conn = _get_connection()
bgneal@519 196
bgneal@519 197 stats = None
bgneal@423 198 if conn:
bgneal@423 199 try:
bgneal@519 200 stats = conn.hgetall(CORE_STATS_KEY)
bgneal@423 201 except redis.RedisError, e:
bgneal@423 202 logger.error(e)
bgneal@423 203
bgneal@519 204 if stats:
bgneal@519 205 return Statistic(
bgneal@519 206 id=1,
bgneal@599 207 max_users=int(stats['max_users']),
bgneal@519 208 max_users_date=datetime.datetime.fromtimestamp(
bgneal@519 209 float(stats['max_users_date'])),
bgneal@599 210 max_anon_users=int(stats['max_anon_users']),
bgneal@519 211 max_anon_users_date=datetime.datetime.fromtimestamp(
bgneal@519 212 float(stats['max_anon_users_date'])))
bgneal@423 213
bgneal@519 214 try:
bgneal@519 215 stats = Statistic.objects.get(pk=1)
bgneal@519 216 except Statistic.DoesNotExist:
bgneal@519 217 return None
bgneal@519 218 else:
bgneal@519 219 _save_stats_to_redis(conn, stats)
bgneal@519 220 return stats
bgneal@519 221
bgneal@519 222
bgneal@519 223 def _save_stats_to_redis(conn, stats):
bgneal@423 224 """
bgneal@519 225 Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
bgneal@519 226 the database from becoming out of sync.
bgneal@519 227
bgneal@423 228 """
bgneal@519 229 fields = dict(
bgneal@519 230 max_users=stats.max_users,
bgneal@519 231 max_users_date=to_timestamp(stats.max_users_date),
bgneal@519 232 max_anon_users=stats.max_anon_users,
bgneal@519 233 max_anon_users_date=to_timestamp(stats.max_anon_users_date))
bgneal@423 234
bgneal@423 235 try:
bgneal@519 236 conn.hmset(CORE_STATS_KEY, fields)
bgneal@519 237 conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
bgneal@423 238 except redis.RedisError, e:
bgneal@423 239 logger.error(e)