Mercurial > public > sg101
view core/whos_online.py @ 943:cf9918328c64
Haystack tweaks for Django 1.7.7.
I had to upgrade to Haystack 2.3.1 to get it to work with Django
1.7.7. I also had to update the Xapian backend. But I ran into
problems.
On my laptop anyway (Ubuntu 14.0.4), xapian gets mad when search terms
are greater than 245 chars (or something) when indexing. So I created
a custom field that would simply omit terms greater than 64 chars and
used this field everywhere I previously used a CharField.
Secondly, the custom search form was broken now. Something changed in
the Xapian backend and exact searches stopped working. Fortunately the
auto_query (which I was using originally and broke during an upgrade)
started working again. So I cut the search form back over to doing an
auto_query. I kept the form the same (3 fields) because I didn't want
to change the form and I think it's better that way.
author | Brian Neal <bgneal@gmail.com> |
---|---|
date | Wed, 13 May 2015 20:25:07 -0500 |
parents | 2f5779e9d8f8 |
children |
line wrap: on
line source
""" This module keeps track of who is online. We maintain records for both authenticated users ("users") and non-authenticated visitors ("visitors"). """ import datetime import logging import time import redis from core.services import get_redis_connection from core.models import Statistic # Users and visitors each have a sorted set in a Redis database. When a user or # visitor is seen, the respective set is updated with the score of the current # time. Periodically we remove elements by score (time) to stale out members. # Redis key names: USER_SET_KEY = "whos_online:users" VISITOR_SET_KEY = "whos_online:visitors" CORE_STATS_KEY = "core:stats" # the period over which we collect who's online stats: MAX_AGE = datetime.timedelta(minutes=15) # Logging: we don't want a Redis malfunction to bring down the site. So we # catch all Redis exceptions, log them, and press on. logger = logging.getLogger(__name__) def _get_connection(): """ Create and return a Redis connection. Returns None on failure. """ try: conn = get_redis_connection() return conn except redis.RedisError, e: logger.error(e) return None def to_timestamp(dt): """ Turn the supplied datetime object into a UNIX timestamp integer. """ return int(time.mktime(dt.timetuple())) def _zadd(key, member): """ Adds the member to the given set key, using the current time as the score. """ conn = _get_connection() if conn: ts = to_timestamp(datetime.datetime.now()) try: conn.zadd(key, ts, member) except redis.RedisError, e: logger.error(e) def _zrangebyscore(key): """ Performs a zrangebyscore operation on the set given by key. The minimum score will be a timestap equal to the current time minus MAX_AGE. The maximum score will be a timestap equal to the current time. """ conn = _get_connection() if conn: now = datetime.datetime.now() min = to_timestamp(now - MAX_AGE) max = to_timestamp(now) try: return conn.zrangebyscore(key, min, max) except redis.RedisError, e: logger.error(e) return [] def report_user(username): """ Call this function when a user has been seen. The username will be added to the set of users online. """ _zadd(USER_SET_KEY, username) def report_visitor(ip): """ Call this function when a visitor has been seen. The IP address will be added to the set of visitors online. """ _zadd(VISITOR_SET_KEY, ip) def get_users_online(): """ Returns a list of user names from the user set. sets. """ return _zrangebyscore(USER_SET_KEY) def get_visitors_online(): """ Returns a list of visitor IP addresses from the visitor set. """ return _zrangebyscore(VISITOR_SET_KEY) def _tick(conn): """ Call this function to "age out" the sets by removing old users/visitors. It then returns a tuple of the form: (zcard users, zcard visitors) """ cutoff = to_timestamp(datetime.datetime.now() - MAX_AGE) try: pipeline = conn.pipeline(transaction=False) pipeline.zremrangebyscore(USER_SET_KEY, 0, cutoff) pipeline.zremrangebyscore(VISITOR_SET_KEY, 0, cutoff) pipeline.zcard(USER_SET_KEY) pipeline.zcard(VISITOR_SET_KEY) result = pipeline.execute() except redis.RedisError, e: logger.error(e) return 0, 0 return result[2], result[3] def max_users(): """ Run this function periodically to clean out the sets and to compute our max users and max visitors statistics. """ conn = _get_connection() if not conn: return num_users, num_visitors = _tick(conn) now = datetime.datetime.now() stats = get_stats(conn) update = False if stats is None: stats = Statistic(id=1, max_users=num_users, max_users_date=now, max_anon_users=num_visitors, max_anon_users_date=now) update = True else: if num_users > stats.max_users: stats.max_users = num_users stats.max_users_date = now update = True if num_visitors > stats.max_anon_users: stats.max_anon_users = num_visitors stats.max_anon_users_date = now update = True if update: _save_stats_to_redis(conn, stats) stats.save() def get_stats(conn=None): """ This function retrieves the who's online max user stats out of Redis. If the keys do not exist in Redis, we fall back to the database. If the stats are not available, None is returned. Note that if we can find stats data, it will be returned as a Statistic object. """ if conn is None: conn = _get_connection() stats = None if conn: try: stats = conn.hgetall(CORE_STATS_KEY) except redis.RedisError, e: logger.error(e) if stats: return Statistic( id=1, max_users=int(stats['max_users']), max_users_date=datetime.datetime.fromtimestamp( float(stats['max_users_date'])), max_anon_users=int(stats['max_anon_users']), max_anon_users_date=datetime.datetime.fromtimestamp( float(stats['max_anon_users_date']))) try: stats = Statistic.objects.get(pk=1) except Statistic.DoesNotExist: return None else: _save_stats_to_redis(conn, stats) return stats def _save_stats_to_redis(conn, stats): """ Saves the statistics to Redis. A TTL is put on the key to prevent Redis and the database from becoming out of sync. """ fields = dict( max_users=stats.max_users, max_users_date=to_timestamp(stats.max_users_date), max_anon_users=stats.max_anon_users, max_anon_users_date=to_timestamp(stats.max_anon_users_date)) try: conn.hmset(CORE_STATS_KEY, fields) conn.expire(CORE_STATS_KEY, 4 * 60 * 60) except redis.RedisError, e: logger.error(e)