Mercurial > public > sg101

comparison gpp/core/whos_online.py @ 519:f72ace06658a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For #194, rework the who's online and max users functions.
author Brian Neal <bgneal@gmail.com>
date Sat, 17 Dec 2011 19:29:24 +0000
parents 6f5fff924877
children
comparison
equal deleted inserted replaced
518:5171a5e9353b 519:f72ace06658a
1 """ 1 """
2 This module keeps track of who is online. We maintain records for both 2 This module keeps track of who is online. We maintain records for both
3 authenticated users ("users") and non-authenticated visitors ("visitors"). 3 authenticated users ("users") and non-authenticated visitors ("visitors").
4 """ 4 """
5 import datetime
5 import logging 6 import logging
7 import time
6 8
7 import redis 9 import redis
8 10
9 from core.services import get_redis_connection 11 from core.services import get_redis_connection
10 12 from core.models import Statistic
11 13
12 # Users and visitors each have 2 sets that we store in a Redis database: 14
13 # a current set and an old set. Whenever a user or visitor is seen, the 15 # Users and visitors each have a sorted set in a Redis database. When a user or
14 # current set is updated. At some interval, the current set is renamed 16 # visitor is seen, the respective set is updated with the score of the current
15 # to the old set, thus destroying it. At any given time, the union of the 17 # time. Periodically we remove elements by score (time) to stale out members.
16 # current and old sets is the "who's online" set.
17 18
18 # Redis key names: 19 # Redis key names:
19 USER_CURRENT_KEY = "wo_user_current" 20 USER_SET_KEY = "whos_online:users"
20 USER_OLD_KEY = "wo_user_old" 21 VISITOR_SET_KEY = "whos_online:visitors"
21 USER_KEYS = [USER_CURRENT_KEY, USER_OLD_KEY] 22
22 23 CORE_STATS_KEY = "core:stats"
23 VISITOR_CURRENT_KEY = "wo_visitor_current" 24
24 VISITOR_OLD_KEY = "wo_visitor_old" 25 # the period over which we collect who's online stats:
25 VISITOR_KEYS = [VISITOR_CURRENT_KEY, VISITOR_OLD_KEY] 26 MAX_AGE = datetime.timedelta(minutes=15)
26 27
27 28
28 # Logging: we don't want a Redis malfunction to bring down the site. So we 29 # Logging: we don't want a Redis malfunction to bring down the site. So we
29 # catch all Redis exceptions, log them, and press on. 30 # catch all Redis exceptions, log them, and press on.
30 logger = logging.getLogger(__name__) 31 logger = logging.getLogger(__name__)
41 logger.error(e) 42 logger.error(e)
42 43
43 return None 44 return None
44 45
45 46
46 def report_user(username): 47 def to_timestamp(dt):
47 """ 48 """
48 Call this function when a user has been seen. The username will be added to 49 Turn the supplied datetime object into a UNIX timestamp integer.
49 the current set. 50
51 """
52 return int(time.mktime(dt.timetuple()))
53
54
55 def _zadd(key, member):
56 """
57 Adds the member to the given set key, using the current time as the score.
58
50 """ 59 """
51 conn = _get_connection() 60 conn = _get_connection()
52 if conn: 61 if conn:
62 ts = to_timestamp(datetime.datetime.now())
53 try: 63 try:
54 conn.sadd(USER_CURRENT_KEY, username) 64 conn.zadd(key, ts, member)
55 except redis.RedisError, e: 65 except redis.RedisError, e:
56 logger.error(e) 66 logger.error(e)
57 67
58 68
59 def report_visitor(ip): 69 def _zrangebyscore(key):
60 """ 70 """
61 Call this function when a visitor has been seen. The IP address will be 71 Performs a zrangebyscore operation on the set given by key.
62 added the current set. 72 The minimum score will be a timestap equal to the current time
73 minus MAX_AGE. The maximum score will be a timestap equal to the
74 current time.
75
63 """ 76 """
64 conn = _get_connection() 77 conn = _get_connection()
65 if conn: 78 if conn:
79 now = datetime.datetime.now()
80 min = to_timestamp(now - MAX_AGE)
81 max = to_timestamp(now)
66 try: 82 try:
67 conn.sadd(VISITOR_CURRENT_KEY, ip) 83 return conn.zrangebyscore(key, min, max)
68 except redis.RedisError, e: 84 except redis.RedisError, e:
69 logger.error(e) 85 logger.error(e)
70 86
87 return []
88
89
90 def report_user(username):
91 """
92 Call this function when a user has been seen. The username will be added to
93 the set of users online.
94
95 """
96 _zadd(USER_SET_KEY, username)
97
98
99 def report_visitor(ip):
100 """
101 Call this function when a visitor has been seen. The IP address will be
102 added to the set of visitors online.
103
104 """
105 _zadd(VISITOR_SET_KEY, ip)
106
71 107
72 def get_users_online(): 108 def get_users_online():
73 """ 109 """
74 Returns a set of user names which is the union of the current and old 110 Returns a list of user names from the user set.
75 sets. 111 sets.
76 """ 112 """
113 return _zrangebyscore(USER_SET_KEY)
114
115
116 def get_visitors_online():
117 """
118 Returns a list of visitor IP addresses from the visitor set.
119 """
120 return _zrangebyscore(VISITOR_SET_KEY)
121
122
123 def _tick(conn):
124 """
125 Call this function to "age out" the sets by removing old users/visitors.
126 It then returns a tuple of the form:
127 (zcard users, zcard visitors)
128
129 """
130 cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
131
132 try:
133 pipeline = conn.pipeline(transaction=False)
134 pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
135 pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
136 pipeline.zcard(USER_SET_KEY)
137 pipeline.zcard(VISITOR_SET_KEY)
138 result = pipeline.execute()
139 except redis.RedisError, e:
140 logger.error(e)
141 return 0, 0
142
143 return result[2], result[3]
144
145
146 def max_users():
147 """
148 Run this function periodically to clean out the sets and to compute our max
149 users and max visitors statistics.
150
151 """
77 conn = _get_connection() 152 conn = _get_connection()
153 if not conn:
154 return
155
156 num_users, num_visitors = _tick(conn)
157 now = datetime.datetime.now()
158
159 stats = get_stats(conn)
160 update = False
161
162 if stats is None:
163 stats = Statistic(id=1,
164 max_users=num_users,
165 max_users_date=now,
166 max_anon_users=num_visitors,
167 max_anon_users_date=now)
168 update = True
169 else:
170 if num_users > stats.max_users:
171 stats.max_users = num_users
172 stats.max_users_date = now
173 update = True
174
175 if num_visitors > stats.max_anon_users:
176 stats.max_anon_users = num_visitors
177 stats.max_anon_users_date = now
178 update = True
179
180 if update:
181 _save_stats_to_redis(conn, stats)
182 stats.save()
183
184
185 def get_stats(conn=None):
186 """
187 This function retrieves the who's online max user stats out of Redis. If
188 the keys do not exist in Redis, we fall back to the database. If the stats
189 are not available, None is returned.
190 Note that if we can find stats data, it will be returned as a Statistic
191 object.
192
193 """
194 if conn is None:
195 conn = _get_connection()
196
197 stats = None
78 if conn: 198 if conn:
79 try: 199 try:
80 # Note that keys that do not exist are considered empty sets 200 stats = conn.hgetall(CORE_STATS_KEY)
81 return conn.sunion(USER_KEYS)
82 except redis.RedisError, e: 201 except redis.RedisError, e:
83 logger.error(e) 202 logger.error(e)
84 203
85 return set() 204 if stats:
86 205 return Statistic(
87 206 id=1,
88 def get_visitors_online(): 207 max_users=stats['max_users'],
89 """ 208 max_users_date=datetime.datetime.fromtimestamp(
90 Returns a set of visitor IP addresses which is the union of the current 209 float(stats['max_users_date'])),
91 and old sets. 210 max_anon_users=stats['max_anon_users'],
92 """ 211 max_anon_users_date=datetime.datetime.fromtimestamp(
93 conn = _get_connection() 212 float(stats['max_anon_users_date'])))
94 if conn: 213
95 try: 214 try:
96 # Note that keys that do not exist are considered empty sets 215 stats = Statistic.objects.get(pk=1)
97 return conn.sunion(VISITOR_KEYS) 216 except Statistic.DoesNotExist:
98 except redis.RedisError, e: 217 return None
99 logger.error(e) 218 else:
100 219 _save_stats_to_redis(conn, stats)
101 return set() 220 return stats
102 221
103 222
104 def _tick_set(conn, current, old): 223 def _save_stats_to_redis(conn, stats):
105 """ 224 """
106 This function renames the set named "current" to "old". 225 Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
107 """ 226 the database from becoming out of sync.
108 # An exception may be raised if the current key doesn't exist; if that 227
109 # happens we have to delete the old set because no one is online. 228 """
110 try: 229 fields = dict(
111 conn.rename(current, old) 230 max_users=stats.max_users,
112 except redis.ResponseError: 231 max_users_date=to_timestamp(stats.max_users_date),
113 try: 232 max_anon_users=stats.max_anon_users,
114 del conn[old] 233 max_anon_users_date=to_timestamp(stats.max_anon_users_date))
115 except redis.RedisError, e: 234
116 logger.error(e) 235 try:
236 conn.hmset(CORE_STATS_KEY, fields)
237 conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
117 except redis.RedisError, e: 238 except redis.RedisError, e:
118 logger.error(e) 239 logger.error(e)
119
120
121 def tick():
122 """
123 Call this function to "age out" the old sets by renaming the current sets
124 to the old.
125 """
126 conn = _get_connection()
127 if conn:
128 _tick_set(conn, USER_CURRENT_KEY, USER_OLD_KEY)
129 _tick_set(conn, VISITOR_CURRENT_KEY, VISITOR_OLD_KEY)