Mercurial > public > sg101

annotate legacy/management/commands/import_old_news_comments.py @ 645:99f7917702ca

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix 081a88b3bfc8, javascript resize of forum images. Commit 081a88b3bfc8 broke those pages that loaded forums.js but did not load the imagesLoaded jQuery extension. Now we have arranged it so that only the forums topic view loads imagesLoaded and put the resizing javascript inline.
author Brian Neal <bgneal@gmail.com>
date Mon, 11 Mar 2013 15:30:25 -0500
parents ee87ea74d46b
children
rev   line source
bgneal@292 1 """
bgneal@292 2 import_old_news_comments.py - For importing comments on news stories from SG101 1.0 as csv files.
bgneal@292 3 """
bgneal@292 4 from __future__ import with_statement
bgneal@292 5 import csv
bgneal@292 6 import optparse
bgneal@292 7 import sys
bgneal@292 8 from datetime import datetime
bgneal@292 9
bgneal@292 10 from django.core.management.base import LabelCommand, CommandError
bgneal@292 11 from django.contrib.auth.models import User
bgneal@292 12 from django.contrib.contenttypes.models import ContentType
bgneal@292 13
bgneal@292 14 from comments.models import Comment
bgneal@292 15 from news.models import Story
bgneal@292 16 import legacy.data
bgneal@292 17 from legacy.html2md import MarkdownWriter
bgneal@292 18
bgneal@292 19
bgneal@292 20 class Command(LabelCommand):
bgneal@292 21 args = '<filename filename ...>'
bgneal@292 22 help = 'Imports news story comments from the old database in CSV format'
bgneal@292 23 option_list = LabelCommand.option_list + (
bgneal@292 24 optparse.make_option("-p", "--progress", action="store_true",
bgneal@292 25 help="Output a . after every 20 comments to show progress"),
bgneal@292 26 )
bgneal@292 27 md_writer = MarkdownWriter()
bgneal@292 28
bgneal@292 29 def handle_label(self, filename, **options):
bgneal@292 30 """
bgneal@292 31 Process each line in the CSV file given by filename by
bgneal@292 32 creating a new story comment.
bgneal@292 33
bgneal@292 34 """
bgneal@292 35 self.show_progress = options.get('progress')
bgneal@292 36 self.users = {}
bgneal@292 37
bgneal@292 38 try:
bgneal@292 39 with open(filename, "rb") as f:
bgneal@292 40 self.reader = csv.DictReader(f)
bgneal@292 41 num_rows = 0
bgneal@292 42 try:
bgneal@292 43 for row in self.reader:
bgneal@292 44 self.process_row(row)
bgneal@292 45 num_rows += 1
bgneal@292 46 if self.show_progress and num_rows % 20 == 0:
bgneal@292 47 sys.stdout.write('.')
bgneal@292 48 sys.stdout.flush()
bgneal@292 49 except csv.Error, e:
bgneal@292 50 raise CommandError("CSV error: %s %s %s" % (
bgneal@292 51 filename, self.reader.line_num, e))
bgneal@292 52
bgneal@292 53 print
bgneal@292 54
bgneal@292 55 except IOError:
bgneal@292 56 raise CommandError("Could not open file: %s" % filename)
bgneal@292 57
bgneal@292 58 def process_row(self, row):
bgneal@292 59 """
bgneal@292 60 Process one row from the CSV file: create a Comment object for
bgneal@292 61 the row and save it in the database.
bgneal@292 62
bgneal@292 63 """
bgneal@292 64 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
bgneal@292 65
bgneal@292 66 try:
bgneal@292 67 user = self._get_user(row['name'])
bgneal@292 68 except User.DoesNotExist:
bgneal@292 69 print "Could not find user %s for comment %s; skipping." % (
bgneal@292 70 row['name'], row['tid'])
bgneal@292 71 return
bgneal@292 72
bgneal@292 73 try:
bgneal@292 74 story = Story.objects.get(id=int(row['sid']))
bgneal@292 75 except Story.DoesNotExist:
bgneal@292 76 print "Could not find story %s for comment %s; skipping." % (
bgneal@292 77 row['sid'], row['tid'])
bgneal@292 78 return
bgneal@292 79
bgneal@292 80 comment = Comment(
bgneal@292 81 id=int(row['tid']),
bgneal@292 82 content_type = ContentType.objects.get_for_model(story),
bgneal@292 83 object_id = story.id,
bgneal@292 84 user = user,
bgneal@292 85 comment = self.to_markdown(row['comment']),
bgneal@292 86 creation_date = datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S"),
bgneal@292 87 ip_address = row['host_name'],
bgneal@292 88 is_public = True,
bgneal@292 89 is_removed = False,
bgneal@292 90 )
bgneal@292 91
bgneal@292 92 comment.save()
bgneal@292 93
bgneal@292 94 def _get_user(self, username):
bgneal@292 95 """
bgneal@292 96 Returns the user object with the given username.
bgneal@292 97 Throws User.DoesNotExist if not found.
bgneal@292 98
bgneal@292 99 """
bgneal@292 100 try:
bgneal@292 101 return self.users[username]
bgneal@292 102 except KeyError:
bgneal@292 103 pass
bgneal@292 104
bgneal@292 105 try:
bgneal@292 106 user = User.objects.get(username=username)
bgneal@292 107 except User.DoesNotExist:
bgneal@535 108 old_name = username.lower()
bgneal@292 109 try:
bgneal@292 110 user = User.objects.get(
bgneal@535 111 username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
bgneal@292 112 except KeyError:
bgneal@292 113 raise User.DoesNotExist
bgneal@292 114
bgneal@292 115 self.users[username] = user
bgneal@292 116 return user
bgneal@292 117
bgneal@292 118 def to_markdown(self, s):
bgneal@292 119 self.md_writer.reset()
bgneal@292 120
bgneal@292 121 if not isinstance(s, unicode):
bgneal@294 122 s = s.decode('latin-1', 'replace')
bgneal@292 123
bgneal@292 124 self.md_writer.feed(s)
bgneal@292 125 return self.md_writer.markdown()