Mercurial > public > sg101

changeset 535:4021ea1045f7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Adding import commands for the 1.0 POTD & comments.
author Brian Neal <bgneal@gmail.com>
date Sat, 31 Dec 2011 21:34:17 +0000
parents 98f5facc0030
children 763eae227302
files gpp/legacy/data.py gpp/legacy/management/commands/import_old_download_comments.py gpp/legacy/management/commands/import_old_news.py gpp/legacy/management/commands/import_old_news_comments.py gpp/legacy/management/commands/import_old_potd.py gpp/legacy/management/commands/import_old_potd_comments.py gpp/potd/admin.py gpp/potd/models.py
diffstat 8 files changed, 282 insertions(+), 11 deletions(-) [+]
line wrap: on
line diff
--- a/gpp/legacy/data.py	Tue Dec 27 01:15:27 2011 +0000
+++ b/gpp/legacy/data.py	Sat Dec 31 21:34:17 2011 +0000
@@ -10,8 +10,18 @@
 
 KNOWN_USERNAME_CHANGES = {
     'cavefishbutchdelux': 'butchdelux',
-    'Findicator1': 'WaveOhhh',
-    'Tikimania': 'Tikitena',
+    'findicator1': 'WaveOhhh',
+    'tikimania': 'Tikitena',
     'sandyfeet': 'RickRhoades',
+    'crumb': 'crumble',
+    'allenbridgewater': 'Outerwave_Allen',
+    'reddtyde': 'Redd_Tyde',
+    'fendershowman63': 'Abe',
+    'hearteater': 'JoshHeartless',
+    'surfdaddy': 'zzero',
+    'frisbie': 'zzero',
+    'retroactivegammarays': 'Retroactive_Taj',
+    'mrrebel': 'Eddie_Bertrand',
+    'doublecoil': 'Showman',
 }
 
--- a/gpp/legacy/management/commands/import_old_download_comments.py	Tue Dec 27 01:15:27 2011 +0000
+++ b/gpp/legacy/management/commands/import_old_download_comments.py	Sat Dec 31 21:34:17 2011 +0000
@@ -59,9 +59,10 @@
         try:
             user = User.objects.get(username=row['ratinguser'])
         except User.DoesNotExist:
+            old_name = row['ratinguser'].lower()
             try:
                 user = User.objects.get(
-                    username=legacy.data.KNOWN_USERNAME_CHANGES[row['ratinguser']])
+                    username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
             except (User.DoesNotExist, KeyError):
                 return
 
--- a/gpp/legacy/management/commands/import_old_news.py	Tue Dec 27 01:15:27 2011 +0000
+++ b/gpp/legacy/management/commands/import_old_news.py	Sat Dec 31 21:34:17 2011 +0000
@@ -110,9 +110,10 @@
         try:
             user = User.objects.get(username=username)
         except User.DoesNotExist:
+            old_name = username.lower()
             try:
                 user = User.objects.get(
-                        username=legacy.data.KNOWN_USERNAME_CHANGES[username])
+                        username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
             except KeyError:
                 raise User.DoesNotExist
 
--- a/gpp/legacy/management/commands/import_old_news_comments.py	Tue Dec 27 01:15:27 2011 +0000
+++ b/gpp/legacy/management/commands/import_old_news_comments.py	Sat Dec 31 21:34:17 2011 +0000
@@ -105,9 +105,10 @@
         try:
             user = User.objects.get(username=username)
         except User.DoesNotExist:
+            old_name = username.lower()
             try:
                 user = User.objects.get(
-                        username=legacy.data.KNOWN_USERNAME_CHANGES[username])
+                        username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
             except KeyError:
                 raise User.DoesNotExist
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/gpp/legacy/management/commands/import_old_potd.py	Sat Dec 31 21:34:17 2011 +0000
@@ -0,0 +1,127 @@
+"""
+import_old_potd.py - For importing POTD's from SG101 1.0 as csv files.
+
+"""
+from __future__ import with_statement
+import csv
+import optparse
+import sys
+from datetime import datetime
+
+from django.core.management.base import LabelCommand, CommandError
+from django.contrib.auth.models import User
+
+from potd.models import Photo
+from legacy.phpbb import unescape
+import legacy.data
+
+
+ID_OFFSET = 100
+
+
+class PathError(Exception):
+    pass
+
+def convert_path(old_path):
+    """
+    Converts the old POTD path to a new one.
+
+    """
+    if old_path.startswith('images/potd/'):
+        return "potd/1.0/%s" % old_path[12:]
+    else:
+        raise PathError("Unknown path %s" % old_path)
+
+
+class Command(LabelCommand):
+    args = '<filename filename ...>'
+    help = "Imports POTD's from the old database in CSV format"
+    option_list = LabelCommand.option_list + (
+        optparse.make_option("-p", "--progress", action="store_true",
+            help="Output a . after every 20 items to show progress"),
+    )
+
+    def handle_label(self, filename, **options):
+        """
+        Process each line in the CSV file given by filename by
+        creating a new Photo
+
+        """
+        self.show_progress = options.get('progress')
+        self.users = {}
+
+        try:
+            with open(filename, "rb") as f:
+                self.reader = csv.DictReader(f)
+                num_rows = 0
+                try:
+                    for row in self.reader:
+                        self.process_row(row)
+                        num_rows += 1
+                        if self.show_progress and num_rows % 20 == 0:
+                            sys.stdout.write('.')
+                            sys.stdout.flush()
+                except csv.Error, e:
+                    raise CommandError("CSV error: %s %s %s" % (
+                        filename, self.reader.line_num, e))
+
+                print
+
+        except IOError:
+            raise CommandError("Could not open file: %s" % filename)
+
+    def process_row(self, row):
+        """
+        Process one row from the CSV file: create a Photo object for
+        the row and save it in the database.
+
+        """
+        try:
+            submitter = self._get_user(row['submitted_by'].decode('latin-1'))
+        except User.DoesNotExist:
+            print "Could not find user %s for potd %s; skipping." % (
+                    row['submitted_by'], row['pid'])
+            return
+
+        desc = row['description'].decode('latin-1').replace('\n', '\n<br />')
+
+        try:
+            photo = Photo(
+                id=int(row['pid']) + ID_OFFSET,
+                photo=convert_path(row['photo_path']),
+                thumb=convert_path(row['thumb_path']),
+                caption=unescape(row['title'].decode('latin-1')),
+                description=desc,
+                user=submitter,
+                date_added=datetime.strptime(row['date_added'],
+                    "%Y-%m-%d %H:%M:%S"),
+                potd_count=int(row['chosen_count']))
+        except PathError, ex:
+            self.stderr.write("\n%s, skipping\n" % ex)
+            return
+
+        photo.save()
+
+    def _get_user(self, username):
+        """
+        Returns the user object with the given username.
+        Throws User.DoesNotExist if not found.
+
+        """
+        try:
+            return self.users[username]
+        except KeyError:
+            pass
+
+        try:
+            user = User.objects.get(username=username)
+        except User.DoesNotExist:
+            old_name = username.lower()
+            try:
+                user = User.objects.get(
+                        username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
+            except KeyError:
+                raise User.DoesNotExist
+
+        self.users[username] = user
+        return user
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/gpp/legacy/management/commands/import_old_potd_comments.py	Sat Dec 31 21:34:17 2011 +0000
@@ -0,0 +1,128 @@
+"""
+import_old_potd_comments.py - For importing comments on POTD's from SG101 1.0
+as csv files.
+
+"""
+from __future__ import with_statement
+import csv
+import optparse
+import sys
+from datetime import datetime
+
+from django.core.management.base import LabelCommand, CommandError
+from django.contrib.auth.models import User
+from django.contrib.contenttypes.models import ContentType
+
+from comments.models import Comment
+from potd.models import Photo
+import legacy.data
+from legacy.html2md import MarkdownWriter
+
+
+PHOTO_ID_OFFSET = 100
+ID_OFFSET = 3000
+
+
+class Command(LabelCommand):
+    args = '<filename filename ...>'
+    help = 'Imports POTD comments from the old database in CSV format'
+    option_list = LabelCommand.option_list + (
+        optparse.make_option("-p", "--progress", action="store_true",
+            help="Output a . after every 20 items to show progress"),
+    )
+    md_writer = MarkdownWriter()
+
+    def handle_label(self, filename, **options):
+        """
+        Process each line in the CSV file given by filename by
+        creating a new POTD comment.
+
+        """
+        self.show_progress = options.get('progress')
+        self.users = {}
+
+        try:
+            with open(filename, "rb") as f:
+                self.reader = csv.DictReader(f)
+                num_rows = 0
+                try:
+                    for row in self.reader:
+                        self.process_row(row)
+                        num_rows += 1
+                        if self.show_progress and num_rows % 20 == 0:
+                            sys.stdout.write('.')
+                            sys.stdout.flush()
+                except csv.Error, e:
+                    raise CommandError("CSV error: %s %s %s" % (
+                        filename, self.reader.line_num, e))
+
+                print
+
+        except IOError:
+            raise CommandError("Could not open file: %s" % filename)
+
+    def process_row(self, row):
+        """
+        Process one row from the CSV file: create a Comment object for
+        the row and save it in the database.
+
+        """
+        try:
+            user = self._get_user(row['username'].decode('latin-1'))
+        except User.DoesNotExist:
+            print "Could not find user %s for comment %s; skipping." % (
+                    row['username'], row['cid'])
+            return
+
+        pid = int(row['pid']) + PHOTO_ID_OFFSET
+        try:
+            photo = Photo.objects.get(id=pid)
+        except Photo.DoesNotExist:
+            print "Could not find photo %s for comment %s; skipping." % (
+                    pid, row['cid'])
+            return
+
+        comment = Comment(
+            id=int(row['cid']) + ID_OFFSET,
+            content_type=ContentType.objects.get_for_model(photo),
+            object_id=photo.id,
+            user=user,
+            comment=self.to_markdown(row['comment'].decode('latin-1')),
+            creation_date=datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S"),
+            ip_address='192.0.2.0',     # TEST-NET
+            is_public=True,
+            is_removed=False,
+        )
+
+        comment.save()
+
+    def _get_user(self, username):
+        """
+        Returns the user object with the given username.
+        Throws User.DoesNotExist if not found.
+
+        """
+        try:
+            return self.users[username]
+        except KeyError:
+            pass
+
+        try:
+            user = User.objects.get(username=username)
+        except User.DoesNotExist:
+            old_name = username.lower()
+            try:
+                user = User.objects.get(
+                        username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
+            except KeyError:
+                raise User.DoesNotExist
+
+        self.users[username] = user
+        return user
+
+    def to_markdown(self, s):
+
+        s = s.replace('\n', '\n<br />')
+        self.md_writer.reset()
+        self.md_writer.feed(s)
+        return self.md_writer.markdown()
--- a/gpp/potd/admin.py	Tue Dec 27 01:15:27 2011 +0000
+++ b/gpp/potd/admin.py	Sat Dec 31 21:34:17 2011 +0000
@@ -13,10 +13,11 @@
 
 
 class PhotoAdmin(admin.ModelAdmin):
-    fields = ('photo', 'caption', 'description', 'user', 'potd_count')
-    raw_id_fields = ('user', )
-    list_display = ('__unicode__', 'thumbnail')
-    actions = ('regen_thumbnail', )
+    fields = ['photo', 'caption', 'description', 'user', 'potd_count']
+    raw_id_fields = ['user']
+    list_display = ['__unicode__', 'thumbnail']
+    actions = ['regen_thumbnail']
+    search_fields = ['caption', 'description']
 
     class Media:
         js = settings.GPP_THIRD_PARTY_JS['tiny_mce']
--- a/gpp/potd/models.py	Tue Dec 27 01:15:27 2011 +0000
+++ b/gpp/potd/models.py	Sat Dec 31 21:34:17 2011 +0000
@@ -2,6 +2,7 @@
 Models for the Photo Of The Day (POTD) application.
 
 """
+import datetime
 import os
 from PIL import ImageFile
 from PIL import Image
@@ -32,7 +33,7 @@
     caption = models.CharField(max_length=128)
     description = models.TextField()
     user = models.ForeignKey(User)
-    date_added = models.DateField(auto_now_add=True)
+    date_added = models.DateField()
     potd_count = models.IntegerField(default=0)
 
     class Meta:
@@ -48,6 +49,7 @@
     def save(self, *args, **kwargs):
         if not self.pk:
             self.generate_thumb()
+            self.date_added = datetime.datetime.now()
 
         super(Photo, self).save(*args, **kwargs)
 
@@ -169,7 +171,7 @@
     objects = SequenceManager()
 
     def __unicode__(self):
-        return self.seq
+        return u'POTD Sequence %d' % self.id
 
     class Meta:
         verbose_name_plural = 'Sequence'