Mercurial > public > sg101

annotate gpp/legacy/management/commands/import_old_potd_comments.py @ 537:eac0ce5e137d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added a "fix-mode" for the potd comment importer command such that it will only create comments if they don't already exist.
author Brian Neal <bgneal@gmail.com>
date Sat, 31 Dec 2011 22:21:58 +0000
parents 4021ea1045f7
children
rev   line source
bgneal@535 1 """
bgneal@535 2 import_old_potd_comments.py - For importing comments on POTD's from SG101 1.0
bgneal@535 3 as csv files.
bgneal@535 4
bgneal@535 5 """
bgneal@535 6 from __future__ import with_statement
bgneal@535 7 import csv
bgneal@535 8 import optparse
bgneal@535 9 import sys
bgneal@535 10 from datetime import datetime
bgneal@535 11
bgneal@535 12 from django.core.management.base import LabelCommand, CommandError
bgneal@535 13 from django.contrib.auth.models import User
bgneal@535 14 from django.contrib.contenttypes.models import ContentType
bgneal@535 15
bgneal@535 16 from comments.models import Comment
bgneal@535 17 from potd.models import Photo
bgneal@535 18 import legacy.data
bgneal@535 19 from legacy.html2md import MarkdownWriter
bgneal@535 20
bgneal@535 21
bgneal@535 22 PHOTO_ID_OFFSET = 100
bgneal@535 23 ID_OFFSET = 3000
bgneal@535 24
bgneal@535 25
bgneal@535 26 class Command(LabelCommand):
bgneal@535 27 args = '<filename filename ...>'
bgneal@535 28 help = 'Imports POTD comments from the old database in CSV format'
bgneal@535 29 option_list = LabelCommand.option_list + (
bgneal@535 30 optparse.make_option("-p", "--progress", action="store_true",
bgneal@535 31 help="Output a . after every 20 items to show progress"),
bgneal@537 32 optparse.make_option("--fix-mode", action="store_true",
bgneal@537 33 help="Only create comments if they don't exist already"),
bgneal@535 34 )
bgneal@535 35 md_writer = MarkdownWriter()
bgneal@535 36
bgneal@535 37 def handle_label(self, filename, **options):
bgneal@535 38 """
bgneal@535 39 Process each line in the CSV file given by filename by
bgneal@535 40 creating a new POTD comment.
bgneal@535 41
bgneal@535 42 """
bgneal@535 43 self.show_progress = options.get('progress')
bgneal@537 44 self.fix_mode = options.get('fix_mode')
bgneal@535 45 self.users = {}
bgneal@535 46
bgneal@535 47 try:
bgneal@535 48 with open(filename, "rb") as f:
bgneal@535 49 self.reader = csv.DictReader(f)
bgneal@535 50 num_rows = 0
bgneal@535 51 try:
bgneal@535 52 for row in self.reader:
bgneal@535 53 self.process_row(row)
bgneal@535 54 num_rows += 1
bgneal@535 55 if self.show_progress and num_rows % 20 == 0:
bgneal@535 56 sys.stdout.write('.')
bgneal@535 57 sys.stdout.flush()
bgneal@535 58 except csv.Error, e:
bgneal@535 59 raise CommandError("CSV error: %s %s %s" % (
bgneal@535 60 filename, self.reader.line_num, e))
bgneal@535 61
bgneal@535 62 print
bgneal@535 63
bgneal@535 64 except IOError:
bgneal@535 65 raise CommandError("Could not open file: %s" % filename)
bgneal@535 66
bgneal@535 67 def process_row(self, row):
bgneal@535 68 """
bgneal@535 69 Process one row from the CSV file: create a Comment object for
bgneal@535 70 the row and save it in the database.
bgneal@535 71
bgneal@535 72 """
bgneal@537 73 comment_id = int(row['cid']) + ID_OFFSET
bgneal@537 74
bgneal@537 75 if self.fix_mode:
bgneal@537 76 try:
bgneal@537 77 c = Comment.objects.get(pk=comment_id)
bgneal@537 78 except Comment.DoesNotExist:
bgneal@537 79 pass
bgneal@537 80 else:
bgneal@537 81 return
bgneal@537 82
bgneal@535 83 try:
bgneal@535 84 user = self._get_user(row['username'].decode('latin-1'))
bgneal@535 85 except User.DoesNotExist:
bgneal@535 86 print "Could not find user %s for comment %s; skipping." % (
bgneal@535 87 row['username'], row['cid'])
bgneal@535 88 return
bgneal@535 89
bgneal@535 90 pid = int(row['pid']) + PHOTO_ID_OFFSET
bgneal@535 91 try:
bgneal@535 92 photo = Photo.objects.get(id=pid)
bgneal@535 93 except Photo.DoesNotExist:
bgneal@535 94 print "Could not find photo %s for comment %s; skipping." % (
bgneal@535 95 pid, row['cid'])
bgneal@535 96 return
bgneal@535 97
bgneal@535 98 comment = Comment(
bgneal@537 99 id=comment_id,
bgneal@535 100 content_type=ContentType.objects.get_for_model(photo),
bgneal@535 101 object_id=photo.id,
bgneal@535 102 user=user,
bgneal@535 103 comment=self.to_markdown(row['comment'].decode('latin-1')),
bgneal@535 104 creation_date=datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S"),
bgneal@535 105 ip_address='192.0.2.0', # TEST-NET
bgneal@535 106 is_public=True,
bgneal@535 107 is_removed=False,
bgneal@535 108 )
bgneal@535 109
bgneal@535 110 comment.save()
bgneal@535 111
bgneal@535 112 def _get_user(self, username):
bgneal@535 113 """
bgneal@535 114 Returns the user object with the given username.
bgneal@535 115 Throws User.DoesNotExist if not found.
bgneal@535 116
bgneal@535 117 """
bgneal@535 118 try:
bgneal@535 119 return self.users[username]
bgneal@535 120 except KeyError:
bgneal@535 121 pass
bgneal@535 122
bgneal@535 123 try:
bgneal@535 124 user = User.objects.get(username=username)
bgneal@535 125 except User.DoesNotExist:
bgneal@535 126 old_name = username.lower()
bgneal@535 127 try:
bgneal@535 128 user = User.objects.get(
bgneal@535 129 username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
bgneal@535 130 except KeyError:
bgneal@535 131 raise User.DoesNotExist
bgneal@535 132
bgneal@535 133 self.users[username] = user
bgneal@535 134 return user
bgneal@535 135
bgneal@535 136 def to_markdown(self, s):
bgneal@535 137
bgneal@535 138 s = s.replace('\n', '\n<br />')
bgneal@535 139 self.md_writer.reset()
bgneal@535 140 self.md_writer.feed(s)
bgneal@535 141 return self.md_writer.markdown()