Mercurial > public > sg101

annotate core/whos_online.py @ 1034:2f36abf65a62

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix UnicodeEncodeError issue in custom search form.
author Brian Neal <bgneal@gmail.com>
date Sat, 26 Dec 2015 15:28:16 -0600
parents 2f5779e9d8f8
children
rev   line source
bgneal@423 1 """
bgneal@423 2 This module keeps track of who is online. We maintain records for both
bgneal@423 3 authenticated users ("users") and non-authenticated visitors ("visitors").
bgneal@423 4 """
bgneal@519 5 import datetime
bgneal@423 6 import logging
bgneal@519 7 import time
bgneal@423 8
bgneal@423 9 import redis
bgneal@423 10
bgneal@508 11 from core.services import get_redis_connection
bgneal@519 12 from core.models import Statistic
bgneal@508 13
bgneal@508 14
bgneal@519 15 # Users and visitors each have a sorted set in a Redis database. When a user or
bgneal@519 16 # visitor is seen, the respective set is updated with the score of the current
bgneal@519 17 # time. Periodically we remove elements by score (time) to stale out members.
bgneal@423 18
bgneal@423 19 # Redis key names:
bgneal@519 20 USER_SET_KEY = "whos_online:users"
bgneal@519 21 VISITOR_SET_KEY = "whos_online:visitors"
bgneal@423 22
bgneal@519 23 CORE_STATS_KEY = "core:stats"
bgneal@519 24
bgneal@519 25 # the period over which we collect who's online stats:
bgneal@519 26 MAX_AGE = datetime.timedelta(minutes=15)
bgneal@423 27
bgneal@423 28
bgneal@423 29 # Logging: we don't want a Redis malfunction to bring down the site. So we
bgneal@423 30 # catch all Redis exceptions, log them, and press on.
bgneal@423 31 logger = logging.getLogger(__name__)
bgneal@423 32
bgneal@423 33
bgneal@423 34 def _get_connection():
bgneal@423 35 """
bgneal@423 36 Create and return a Redis connection. Returns None on failure.
bgneal@423 37 """
bgneal@423 38 try:
bgneal@508 39 conn = get_redis_connection()
bgneal@423 40 return conn
bgneal@423 41 except redis.RedisError, e:
bgneal@423 42 logger.error(e)
bgneal@423 43
bgneal@423 44 return None
bgneal@423 45
bgneal@423 46
bgneal@519 47 def to_timestamp(dt):
bgneal@519 48 """
bgneal@519 49 Turn the supplied datetime object into a UNIX timestamp integer.
bgneal@519 50
bgneal@519 51 """
bgneal@519 52 return int(time.mktime(dt.timetuple()))
bgneal@519 53
bgneal@519 54
bgneal@519 55 def _zadd(key, member):
bgneal@519 56 """
bgneal@519 57 Adds the member to the given set key, using the current time as the score.
bgneal@519 58
bgneal@519 59 """
bgneal@519 60 conn = _get_connection()
bgneal@519 61 if conn:
bgneal@519 62 ts = to_timestamp(datetime.datetime.now())
bgneal@519 63 try:
bgneal@519 64 conn.zadd(key, ts, member)
bgneal@519 65 except redis.RedisError, e:
bgneal@519 66 logger.error(e)
bgneal@519 67
bgneal@519 68
bgneal@519 69 def _zrangebyscore(key):
bgneal@519 70 """
bgneal@519 71 Performs a zrangebyscore operation on the set given by key.
bgneal@519 72 The minimum score will be a timestap equal to the current time
bgneal@519 73 minus MAX_AGE. The maximum score will be a timestap equal to the
bgneal@519 74 current time.
bgneal@519 75
bgneal@519 76 """
bgneal@519 77 conn = _get_connection()
bgneal@519 78 if conn:
bgneal@519 79 now = datetime.datetime.now()
bgneal@519 80 min = to_timestamp(now - MAX_AGE)
bgneal@519 81 max = to_timestamp(now)
bgneal@519 82 try:
bgneal@519 83 return conn.zrangebyscore(key, min, max)
bgneal@519 84 except redis.RedisError, e:
bgneal@519 85 logger.error(e)
bgneal@519 86
bgneal@519 87 return []
bgneal@519 88
bgneal@519 89
bgneal@423 90 def report_user(username):
bgneal@423 91 """
bgneal@423 92 Call this function when a user has been seen. The username will be added to
bgneal@519 93 the set of users online.
bgneal@519 94
bgneal@423 95 """
bgneal@519 96 _zadd(USER_SET_KEY, username)
bgneal@423 97
bgneal@423 98
bgneal@423 99 def report_visitor(ip):
bgneal@423 100 """
bgneal@423 101 Call this function when a visitor has been seen. The IP address will be
bgneal@519 102 added to the set of visitors online.
bgneal@519 103
bgneal@519 104 """
bgneal@519 105 _zadd(VISITOR_SET_KEY, ip)
bgneal@519 106
bgneal@519 107
bgneal@519 108 def get_users_online():
bgneal@519 109 """
bgneal@519 110 Returns a list of user names from the user set.
bgneal@519 111 sets.
bgneal@519 112 """
bgneal@519 113 return _zrangebyscore(USER_SET_KEY)
bgneal@519 114
bgneal@519 115
bgneal@519 116 def get_visitors_online():
bgneal@519 117 """
bgneal@519 118 Returns a list of visitor IP addresses from the visitor set.
bgneal@519 119 """
bgneal@519 120 return _zrangebyscore(VISITOR_SET_KEY)
bgneal@519 121
bgneal@519 122
bgneal@519 123 def _tick(conn):
bgneal@519 124 """
bgneal@519 125 Call this function to "age out" the sets by removing old users/visitors.
bgneal@519 126 It then returns a tuple of the form:
bgneal@519 127 (zcard users, zcard visitors)
bgneal@519 128
bgneal@519 129 """
bgneal@519 130 cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
bgneal@519 131
bgneal@519 132 try:
bgneal@519 133 pipeline = conn.pipeline(transaction=False)
bgneal@519 134 pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
bgneal@519 135 pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
bgneal@519 136 pipeline.zcard(USER_SET_KEY)
bgneal@519 137 pipeline.zcard(VISITOR_SET_KEY)
bgneal@519 138 result = pipeline.execute()
bgneal@519 139 except redis.RedisError, e:
bgneal@519 140 logger.error(e)
bgneal@519 141 return 0, 0
bgneal@519 142
bgneal@519 143 return result[2], result[3]
bgneal@519 144
bgneal@519 145
bgneal@519 146 def max_users():
bgneal@519 147 """
bgneal@519 148 Run this function periodically to clean out the sets and to compute our max
bgneal@519 149 users and max visitors statistics.
bgneal@519 150
bgneal@423 151 """
bgneal@423 152 conn = _get_connection()
bgneal@519 153 if not conn:
bgneal@519 154 return
bgneal@519 155
bgneal@519 156 num_users, num_visitors = _tick(conn)
bgneal@519 157 now = datetime.datetime.now()
bgneal@519 158
bgneal@519 159 stats = get_stats(conn)
bgneal@519 160 update = False
bgneal@519 161
bgneal@519 162 if stats is None:
bgneal@519 163 stats = Statistic(id=1,
bgneal@519 164 max_users=num_users,
bgneal@519 165 max_users_date=now,
bgneal@519 166 max_anon_users=num_visitors,
bgneal@519 167 max_anon_users_date=now)
bgneal@519 168 update = True
bgneal@519 169 else:
bgneal@519 170 if num_users > stats.max_users:
bgneal@519 171 stats.max_users = num_users
bgneal@519 172 stats.max_users_date = now
bgneal@519 173 update = True
bgneal@519 174
bgneal@519 175 if num_visitors > stats.max_anon_users:
bgneal@519 176 stats.max_anon_users = num_visitors
bgneal@519 177 stats.max_anon_users_date = now
bgneal@519 178 update = True
bgneal@519 179
bgneal@519 180 if update:
bgneal@519 181 _save_stats_to_redis(conn, stats)
bgneal@519 182 stats.save()
bgneal@519 183
bgneal@519 184
bgneal@519 185 def get_stats(conn=None):
bgneal@519 186 """
bgneal@519 187 This function retrieves the who's online max user stats out of Redis. If
bgneal@519 188 the keys do not exist in Redis, we fall back to the database. If the stats
bgneal@519 189 are not available, None is returned.
bgneal@519 190 Note that if we can find stats data, it will be returned as a Statistic
bgneal@519 191 object.
bgneal@519 192
bgneal@519 193 """
bgneal@519 194 if conn is None:
bgneal@519 195 conn = _get_connection()
bgneal@519 196
bgneal@519 197 stats = None
bgneal@423 198 if conn:
bgneal@423 199 try:
bgneal@519 200 stats = conn.hgetall(CORE_STATS_KEY)
bgneal@423 201 except redis.RedisError, e:
bgneal@423 202 logger.error(e)
bgneal@423 203
bgneal@519 204 if stats:
bgneal@519 205 return Statistic(
bgneal@519 206 id=1,
bgneal@599 207 max_users=int(stats['max_users']),
bgneal@519 208 max_users_date=datetime.datetime.fromtimestamp(
bgneal@519 209 float(stats['max_users_date'])),
bgneal@599 210 max_anon_users=int(stats['max_anon_users']),
bgneal@519 211 max_anon_users_date=datetime.datetime.fromtimestamp(
bgneal@519 212 float(stats['max_anon_users_date'])))
bgneal@423 213
bgneal@519 214 try:
bgneal@519 215 stats = Statistic.objects.get(pk=1)
bgneal@519 216 except Statistic.DoesNotExist:
bgneal@519 217 return None
bgneal@519 218 else:
bgneal@519 219 _save_stats_to_redis(conn, stats)
bgneal@519 220 return stats
bgneal@519 221
bgneal@519 222
bgneal@519 223 def _save_stats_to_redis(conn, stats):
bgneal@423 224 """
bgneal@519 225 Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
bgneal@519 226 the database from becoming out of sync.
bgneal@519 227
bgneal@423 228 """
bgneal@519 229 fields = dict(
bgneal@519 230 max_users=stats.max_users,
bgneal@519 231 max_users_date=to_timestamp(stats.max_users_date),
bgneal@519 232 max_anon_users=stats.max_anon_users,
bgneal@519 233 max_anon_users_date=to_timestamp(stats.max_anon_users_date))
bgneal@423 234
bgneal@423 235 try:
bgneal@519 236 conn.hmset(CORE_STATS_KEY, fields)
bgneal@519 237 conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
bgneal@423 238 except redis.RedisError, e:
bgneal@423 239 logger.error(e)