Mercurial > public > sg101
changeset 535:4021ea1045f7
Adding import commands for the 1.0 POTD & comments.
author | Brian Neal <bgneal@gmail.com> |
---|---|
date | Sat, 31 Dec 2011 21:34:17 +0000 (2011-12-31) |
parents | 98f5facc0030 |
children | 763eae227302 |
files | gpp/legacy/data.py gpp/legacy/management/commands/import_old_download_comments.py gpp/legacy/management/commands/import_old_news.py gpp/legacy/management/commands/import_old_news_comments.py gpp/legacy/management/commands/import_old_potd.py gpp/legacy/management/commands/import_old_potd_comments.py gpp/potd/admin.py gpp/potd/models.py |
diffstat | 8 files changed, 282 insertions(+), 11 deletions(-) [+] |
line wrap: on
line diff
--- a/gpp/legacy/data.py Tue Dec 27 01:15:27 2011 +0000 +++ b/gpp/legacy/data.py Sat Dec 31 21:34:17 2011 +0000 @@ -10,8 +10,18 @@ KNOWN_USERNAME_CHANGES = { 'cavefishbutchdelux': 'butchdelux', - 'Findicator1': 'WaveOhhh', - 'Tikimania': 'Tikitena', + 'findicator1': 'WaveOhhh', + 'tikimania': 'Tikitena', 'sandyfeet': 'RickRhoades', + 'crumb': 'crumble', + 'allenbridgewater': 'Outerwave_Allen', + 'reddtyde': 'Redd_Tyde', + 'fendershowman63': 'Abe', + 'hearteater': 'JoshHeartless', + 'surfdaddy': 'zzero', + 'frisbie': 'zzero', + 'retroactivegammarays': 'Retroactive_Taj', + 'mrrebel': 'Eddie_Bertrand', + 'doublecoil': 'Showman', }
--- a/gpp/legacy/management/commands/import_old_download_comments.py Tue Dec 27 01:15:27 2011 +0000 +++ b/gpp/legacy/management/commands/import_old_download_comments.py Sat Dec 31 21:34:17 2011 +0000 @@ -59,9 +59,10 @@ try: user = User.objects.get(username=row['ratinguser']) except User.DoesNotExist: + old_name = row['ratinguser'].lower() try: user = User.objects.get( - username=legacy.data.KNOWN_USERNAME_CHANGES[row['ratinguser']]) + username=legacy.data.KNOWN_USERNAME_CHANGES[old_name]) except (User.DoesNotExist, KeyError): return
--- a/gpp/legacy/management/commands/import_old_news.py Tue Dec 27 01:15:27 2011 +0000 +++ b/gpp/legacy/management/commands/import_old_news.py Sat Dec 31 21:34:17 2011 +0000 @@ -110,9 +110,10 @@ try: user = User.objects.get(username=username) except User.DoesNotExist: + old_name = username.lower() try: user = User.objects.get( - username=legacy.data.KNOWN_USERNAME_CHANGES[username]) + username=legacy.data.KNOWN_USERNAME_CHANGES[old_name]) except KeyError: raise User.DoesNotExist
--- a/gpp/legacy/management/commands/import_old_news_comments.py Tue Dec 27 01:15:27 2011 +0000 +++ b/gpp/legacy/management/commands/import_old_news_comments.py Sat Dec 31 21:34:17 2011 +0000 @@ -105,9 +105,10 @@ try: user = User.objects.get(username=username) except User.DoesNotExist: + old_name = username.lower() try: user = User.objects.get( - username=legacy.data.KNOWN_USERNAME_CHANGES[username]) + username=legacy.data.KNOWN_USERNAME_CHANGES[old_name]) except KeyError: raise User.DoesNotExist
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gpp/legacy/management/commands/import_old_potd.py Sat Dec 31 21:34:17 2011 +0000 @@ -0,0 +1,127 @@ +""" +import_old_potd.py - For importing POTD's from SG101 1.0 as csv files. + +""" +from __future__ import with_statement +import csv +import optparse +import sys +from datetime import datetime + +from django.core.management.base import LabelCommand, CommandError +from django.contrib.auth.models import User + +from potd.models import Photo +from legacy.phpbb import unescape +import legacy.data + + +ID_OFFSET = 100 + + +class PathError(Exception): + pass + +def convert_path(old_path): + """ + Converts the old POTD path to a new one. + + """ + if old_path.startswith('images/potd/'): + return "potd/1.0/%s" % old_path[12:] + else: + raise PathError("Unknown path %s" % old_path) + + +class Command(LabelCommand): + args = '<filename filename ...>' + help = "Imports POTD's from the old database in CSV format" + option_list = LabelCommand.option_list + ( + optparse.make_option("-p", "--progress", action="store_true", + help="Output a . after every 20 items to show progress"), + ) + + def handle_label(self, filename, **options): + """ + Process each line in the CSV file given by filename by + creating a new Photo + + """ + self.show_progress = options.get('progress') + self.users = {} + + try: + with open(filename, "rb") as f: + self.reader = csv.DictReader(f) + num_rows = 0 + try: + for row in self.reader: + self.process_row(row) + num_rows += 1 + if self.show_progress and num_rows % 20 == 0: + sys.stdout.write('.') + sys.stdout.flush() + except csv.Error, e: + raise CommandError("CSV error: %s %s %s" % ( + filename, self.reader.line_num, e)) + + print + + except IOError: + raise CommandError("Could not open file: %s" % filename) + + def process_row(self, row): + """ + Process one row from the CSV file: create a Photo object for + the row and save it in the database. + + """ + try: + submitter = self._get_user(row['submitted_by'].decode('latin-1')) + except User.DoesNotExist: + print "Could not find user %s for potd %s; skipping." % ( + row['submitted_by'], row['pid']) + return + + desc = row['description'].decode('latin-1').replace('\n', '\n<br />') + + try: + photo = Photo( + id=int(row['pid']) + ID_OFFSET, + photo=convert_path(row['photo_path']), + thumb=convert_path(row['thumb_path']), + caption=unescape(row['title'].decode('latin-1')), + description=desc, + user=submitter, + date_added=datetime.strptime(row['date_added'], + "%Y-%m-%d %H:%M:%S"), + potd_count=int(row['chosen_count'])) + except PathError, ex: + self.stderr.write("\n%s, skipping\n" % ex) + return + + photo.save() + + def _get_user(self, username): + """ + Returns the user object with the given username. + Throws User.DoesNotExist if not found. + + """ + try: + return self.users[username] + except KeyError: + pass + + try: + user = User.objects.get(username=username) + except User.DoesNotExist: + old_name = username.lower() + try: + user = User.objects.get( + username=legacy.data.KNOWN_USERNAME_CHANGES[old_name]) + except KeyError: + raise User.DoesNotExist + + self.users[username] = user + return user
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gpp/legacy/management/commands/import_old_potd_comments.py Sat Dec 31 21:34:17 2011 +0000 @@ -0,0 +1,128 @@ +""" +import_old_potd_comments.py - For importing comments on POTD's from SG101 1.0 +as csv files. + +""" +from __future__ import with_statement +import csv +import optparse +import sys +from datetime import datetime + +from django.core.management.base import LabelCommand, CommandError +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType + +from comments.models import Comment +from potd.models import Photo +import legacy.data +from legacy.html2md import MarkdownWriter + + +PHOTO_ID_OFFSET = 100 +ID_OFFSET = 3000 + + +class Command(LabelCommand): + args = '<filename filename ...>' + help = 'Imports POTD comments from the old database in CSV format' + option_list = LabelCommand.option_list + ( + optparse.make_option("-p", "--progress", action="store_true", + help="Output a . after every 20 items to show progress"), + ) + md_writer = MarkdownWriter() + + def handle_label(self, filename, **options): + """ + Process each line in the CSV file given by filename by + creating a new POTD comment. + + """ + self.show_progress = options.get('progress') + self.users = {} + + try: + with open(filename, "rb") as f: + self.reader = csv.DictReader(f) + num_rows = 0 + try: + for row in self.reader: + self.process_row(row) + num_rows += 1 + if self.show_progress and num_rows % 20 == 0: + sys.stdout.write('.') + sys.stdout.flush() + except csv.Error, e: + raise CommandError("CSV error: %s %s %s" % ( + filename, self.reader.line_num, e)) + + print + + except IOError: + raise CommandError("Could not open file: %s" % filename) + + def process_row(self, row): + """ + Process one row from the CSV file: create a Comment object for + the row and save it in the database. + + """ + try: + user = self._get_user(row['username'].decode('latin-1')) + except User.DoesNotExist: + print "Could not find user %s for comment %s; skipping." % ( + row['username'], row['cid']) + return + + pid = int(row['pid']) + PHOTO_ID_OFFSET + try: + photo = Photo.objects.get(id=pid) + except Photo.DoesNotExist: + print "Could not find photo %s for comment %s; skipping." % ( + pid, row['cid']) + return + + comment = Comment( + id=int(row['cid']) + ID_OFFSET, + content_type=ContentType.objects.get_for_model(photo), + object_id=photo.id, + user=user, + comment=self.to_markdown(row['comment'].decode('latin-1')), + creation_date=datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S"), + ip_address='192.0.2.0', # TEST-NET + is_public=True, + is_removed=False, + ) + + comment.save() + + def _get_user(self, username): + """ + Returns the user object with the given username. + Throws User.DoesNotExist if not found. + + """ + try: + return self.users[username] + except KeyError: + pass + + try: + user = User.objects.get(username=username) + except User.DoesNotExist: + old_name = username.lower() + try: + user = User.objects.get( + username=legacy.data.KNOWN_USERNAME_CHANGES[old_name]) + except KeyError: + raise User.DoesNotExist + + self.users[username] = user + return user + + def to_markdown(self, s): + + s = s.replace('\n', '\n<br />') + self.md_writer.reset() + self.md_writer.feed(s) + return self.md_writer.markdown()
--- a/gpp/potd/admin.py Tue Dec 27 01:15:27 2011 +0000 +++ b/gpp/potd/admin.py Sat Dec 31 21:34:17 2011 +0000 @@ -13,10 +13,11 @@ class PhotoAdmin(admin.ModelAdmin): - fields = ('photo', 'caption', 'description', 'user', 'potd_count') - raw_id_fields = ('user', ) - list_display = ('__unicode__', 'thumbnail') - actions = ('regen_thumbnail', ) + fields = ['photo', 'caption', 'description', 'user', 'potd_count'] + raw_id_fields = ['user'] + list_display = ['__unicode__', 'thumbnail'] + actions = ['regen_thumbnail'] + search_fields = ['caption', 'description'] class Media: js = settings.GPP_THIRD_PARTY_JS['tiny_mce']
--- a/gpp/potd/models.py Tue Dec 27 01:15:27 2011 +0000 +++ b/gpp/potd/models.py Sat Dec 31 21:34:17 2011 +0000 @@ -2,6 +2,7 @@ Models for the Photo Of The Day (POTD) application. """ +import datetime import os from PIL import ImageFile from PIL import Image @@ -32,7 +33,7 @@ caption = models.CharField(max_length=128) description = models.TextField() user = models.ForeignKey(User) - date_added = models.DateField(auto_now_add=True) + date_added = models.DateField() potd_count = models.IntegerField(default=0) class Meta: @@ -48,6 +49,7 @@ def save(self, *args, **kwargs): if not self.pk: self.generate_thumb() + self.date_added = datetime.datetime.now() super(Photo, self).save(*args, **kwargs) @@ -169,7 +171,7 @@ objects = SequenceManager() def __unicode__(self): - return self.seq + return u'POTD Sequence %d' % self.id class Meta: verbose_name_plural = 'Sequence'