Mercurial > public > sg101

comparison gpp/legacy/management/commands/import_old_news_comments.py @ 292:2367c4795c92

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added a legacy management command to import old news comments.
author Brian Neal <bgneal@gmail.com>
date Fri, 24 Dec 2010 22:20:30 +0000
parents
children 254db4cb6a86
comparison
equal deleted inserted replaced
291:a6357f2bcbbc 292:2367c4795c92
1 """
2 import_old_news_comments.py - For importing comments on news stories from SG101 1.0 as csv files.
3 """
4 from __future__ import with_statement
5 import csv
6 import optparse
7 import sys
8 from datetime import datetime
9
10 from django.core.management.base import LabelCommand, CommandError
11 from django.contrib.auth.models import User
12 from django.contrib.contenttypes.models import ContentType
13
14 from comments.models import Comment
15 from news.models import Story
16 import legacy.data
17 from legacy.html2md import MarkdownWriter
18
19
20 class Command(LabelCommand):
21 args = '<filename filename ...>'
22 help = 'Imports news story comments from the old database in CSV format'
23 option_list = LabelCommand.option_list + (
24 optparse.make_option("-p", "--progress", action="store_true",
25 help="Output a . after every 20 comments to show progress"),
26 )
27 md_writer = MarkdownWriter()
28
29 def handle_label(self, filename, **options):
30 """
31 Process each line in the CSV file given by filename by
32 creating a new story comment.
33
34 """
35 self.show_progress = options.get('progress')
36 self.users = {}
37
38 try:
39 with open(filename, "rb") as f:
40 self.reader = csv.DictReader(f)
41 num_rows = 0
42 try:
43 for row in self.reader:
44 self.process_row(row)
45 num_rows += 1
46 if self.show_progress and num_rows % 20 == 0:
47 sys.stdout.write('.')
48 sys.stdout.flush()
49 except csv.Error, e:
50 raise CommandError("CSV error: %s %s %s" % (
51 filename, self.reader.line_num, e))
52
53 print
54
55 except IOError:
56 raise CommandError("Could not open file: %s" % filename)
57
58 def process_row(self, row):
59 """
60 Process one row from the CSV file: create a Comment object for
61 the row and save it in the database.
62
63 """
64 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
65
66 try:
67 user = self._get_user(row['name'])
68 except User.DoesNotExist:
69 print "Could not find user %s for comment %s; skipping." % (
70 row['name'], row['tid'])
71 return
72
73 try:
74 story = Story.objects.get(id=int(row['sid']))
75 except Story.DoesNotExist:
76 print "Could not find story %s for comment %s; skipping." % (
77 row['sid'], row['tid'])
78 return
79
80 comment = Comment(
81 id=int(row['tid']),
82 content_type = ContentType.objects.get_for_model(story),
83 object_id = story.id,
84 user = user,
85 comment = self.to_markdown(row['comment']),
86 creation_date = datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S"),
87 ip_address = row['host_name'],
88 is_public = True,
89 is_removed = False,
90 )
91
92 comment.save()
93
94 def _get_user(self, username):
95 """
96 Returns the user object with the given username.
97 Throws User.DoesNotExist if not found.
98
99 """
100 try:
101 return self.users[username]
102 except KeyError:
103 pass
104
105 try:
106 user = User.objects.get(username=username)
107 except User.DoesNotExist:
108 try:
109 user = User.objects.get(
110 username=legacy.data.KNOWN_USERNAME_CHANGES[username])
111 except KeyError:
112 raise User.DoesNotExist
113
114 self.users[username] = user
115 return user
116
117 def to_markdown(self, s):
118 self.md_writer.reset()
119
120 if not isinstance(s, unicode):
121 s = s.decode('utf-8', 'replace')
122
123 self.md_writer.feed(s)
124 return self.md_writer.markdown()