Mercurial > public > sg101

comparison core/whos_online.py @ 581:ee87ea74d46b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For Django 1.4, rearranged project structure for new manage.py.
author Brian Neal <bgneal@gmail.com>
date Sat, 05 May 2012 17:10:48 -0500
parents gpp/core/whos_online.py@f72ace06658a
children 2f5779e9d8f8
comparison
equal deleted inserted replaced
580:c525f3e0b5d0 581:ee87ea74d46b
1 """
2 This module keeps track of who is online. We maintain records for both
3 authenticated users ("users") and non-authenticated visitors ("visitors").
4 """
5 import datetime
6 import logging
7 import time
8
9 import redis
10
11 from core.services import get_redis_connection
12 from core.models import Statistic
13
14
15 # Users and visitors each have a sorted set in a Redis database. When a user or
16 # visitor is seen, the respective set is updated with the score of the current
17 # time. Periodically we remove elements by score (time) to stale out members.
18
19 # Redis key names:
20 USER_SET_KEY = "whos_online:users"
21 VISITOR_SET_KEY = "whos_online:visitors"
22
23 CORE_STATS_KEY = "core:stats"
24
25 # the period over which we collect who's online stats:
26 MAX_AGE = datetime.timedelta(minutes=15)
27
28
29 # Logging: we don't want a Redis malfunction to bring down the site. So we
30 # catch all Redis exceptions, log them, and press on.
31 logger = logging.getLogger(__name__)
32
33
34 def _get_connection():
35 """
36 Create and return a Redis connection. Returns None on failure.
37 """
38 try:
39 conn = get_redis_connection()
40 return conn
41 except redis.RedisError, e:
42 logger.error(e)
43
44 return None
45
46
47 def to_timestamp(dt):
48 """
49 Turn the supplied datetime object into a UNIX timestamp integer.
50
51 """
52 return int(time.mktime(dt.timetuple()))
53
54
55 def _zadd(key, member):
56 """
57 Adds the member to the given set key, using the current time as the score.
58
59 """
60 conn = _get_connection()
61 if conn:
62 ts = to_timestamp(datetime.datetime.now())
63 try:
64 conn.zadd(key, ts, member)
65 except redis.RedisError, e:
66 logger.error(e)
67
68
69 def _zrangebyscore(key):
70 """
71 Performs a zrangebyscore operation on the set given by key.
72 The minimum score will be a timestap equal to the current time
73 minus MAX_AGE. The maximum score will be a timestap equal to the
74 current time.
75
76 """
77 conn = _get_connection()
78 if conn:
79 now = datetime.datetime.now()
80 min = to_timestamp(now - MAX_AGE)
81 max = to_timestamp(now)
82 try:
83 return conn.zrangebyscore(key, min, max)
84 except redis.RedisError, e:
85 logger.error(e)
86
87 return []
88
89
90 def report_user(username):
91 """
92 Call this function when a user has been seen. The username will be added to
93 the set of users online.
94
95 """
96 _zadd(USER_SET_KEY, username)
97
98
99 def report_visitor(ip):
100 """
101 Call this function when a visitor has been seen. The IP address will be
102 added to the set of visitors online.
103
104 """
105 _zadd(VISITOR_SET_KEY, ip)
106
107
108 def get_users_online():
109 """
110 Returns a list of user names from the user set.
111 sets.
112 """
113 return _zrangebyscore(USER_SET_KEY)
114
115
116 def get_visitors_online():
117 """
118 Returns a list of visitor IP addresses from the visitor set.
119 """
120 return _zrangebyscore(VISITOR_SET_KEY)
121
122
123 def _tick(conn):
124 """
125 Call this function to "age out" the sets by removing old users/visitors.
126 It then returns a tuple of the form:
127 (zcard users, zcard visitors)
128
129 """
130 cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE)
131
132 try:
133 pipeline = conn.pipeline(transaction=False)
134 pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff)
135 pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff)
136 pipeline.zcard(USER_SET_KEY)
137 pipeline.zcard(VISITOR_SET_KEY)
138 result = pipeline.execute()
139 except redis.RedisError, e:
140 logger.error(e)
141 return 0, 0
142
143 return result[2], result[3]
144
145
146 def max_users():
147 """
148 Run this function periodically to clean out the sets and to compute our max
149 users and max visitors statistics.
150
151 """
152 conn = _get_connection()
153 if not conn:
154 return
155
156 num_users, num_visitors = _tick(conn)
157 now = datetime.datetime.now()
158
159 stats = get_stats(conn)
160 update = False
161
162 if stats is None:
163 stats = Statistic(id=1,
164 max_users=num_users,
165 max_users_date=now,
166 max_anon_users=num_visitors,
167 max_anon_users_date=now)
168 update = True
169 else:
170 if num_users > stats.max_users:
171 stats.max_users = num_users
172 stats.max_users_date = now
173 update = True
174
175 if num_visitors > stats.max_anon_users:
176 stats.max_anon_users = num_visitors
177 stats.max_anon_users_date = now
178 update = True
179
180 if update:
181 _save_stats_to_redis(conn, stats)
182 stats.save()
183
184
185 def get_stats(conn=None):
186 """
187 This function retrieves the who's online max user stats out of Redis. If
188 the keys do not exist in Redis, we fall back to the database. If the stats
189 are not available, None is returned.
190 Note that if we can find stats data, it will be returned as a Statistic
191 object.
192
193 """
194 if conn is None:
195 conn = _get_connection()
196
197 stats = None
198 if conn:
199 try:
200 stats = conn.hgetall(CORE_STATS_KEY)
201 except redis.RedisError, e:
202 logger.error(e)
203
204 if stats:
205 return Statistic(
206 id=1,
207 max_users=stats['max_users'],
208 max_users_date=datetime.datetime.fromtimestamp(
209 float(stats['max_users_date'])),
210 max_anon_users=stats['max_anon_users'],
211 max_anon_users_date=datetime.datetime.fromtimestamp(
212 float(stats['max_anon_users_date'])))
213
214 try:
215 stats = Statistic.objects.get(pk=1)
216 except Statistic.DoesNotExist:
217 return None
218 else:
219 _save_stats_to_redis(conn, stats)
220 return stats
221
222
223 def _save_stats_to_redis(conn, stats):
224 """
225 Saves the statistics to Redis. A TTL is put on the key to prevent Redis and
226 the database from becoming out of sync.
227
228 """
229 fields = dict(
230 max_users=stats.max_users,
231 max_users_date=to_timestamp(stats.max_users_date),
232 max_anon_users=stats.max_anon_users,
233 max_anon_users_date=to_timestamp(stats.max_anon_users_date))
234
235 try:
236 conn.hmset(CORE_STATS_KEY, fields)
237 conn.expire(CORE_STATS_KEY, 4 * 60 * 60)
238 except redis.RedisError, e:
239 logger.error(e)