Mercurial > public > sg101

comparison legacy/management/commands/import_old_topics.py @ 581:ee87ea74d46b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For Django 1.4, rearranged project structure for new manage.py.
author Brian Neal <bgneal@gmail.com>
date Sat, 05 May 2012 17:10:48 -0500
parents gpp/legacy/management/commands/import_old_topics.py@254db4cb6a86
children
comparison
equal deleted inserted replaced
580:c525f3e0b5d0 581:ee87ea74d46b
1 """
2 import_old_topics.py - For importing forum topics (threads) from SG101 1.0 as
3 csv files.
4
5 """
6 from __future__ import with_statement
7 import csv
8 import optparse
9 import sys
10 from datetime import datetime
11
12 from django.core.management.base import LabelCommand, CommandError
13 from django.contrib.auth.models import User
14
15 from forums.models import Forum, Topic
16 from legacy.phpbb import unescape
17
18
19 class Command(LabelCommand):
20 args = '<filename filename ...>'
21 help = 'Imports forum topics from the old database in CSV format'
22 option_list = LabelCommand.option_list + (
23 optparse.make_option("-p", "--progress", action="store_true",
24 help="Output a . after every 20 topics to show progress"),
25 )
26
27 def handle_label(self, filename, **options):
28 """
29 Process each line in the CSV file given by filename by
30 creating a new topic.
31
32 """
33 self.show_progress = options.get('progress')
34 self.users = {}
35
36 # Create a mapping from the old database's forums to our
37 # forums
38 self.forums = {}
39 try:
40 self.forums[2] = Forum.objects.get(slug='suggestion-box')
41 self.forums[3] = Forum.objects.get(slug='surf-music')
42 self.forums[4] = Forum.objects.get(slug='surf-musician')
43 self.forums[5] = Forum.objects.get(slug='gear')
44 self.forums[6] = Forum.objects.get(slug='recording-corner')
45 self.forums[7] = Forum.objects.get(slug='shallow-end')
46 self.forums[8] = Forum.objects.get(slug='surfguitar101-website')
47 self.forums[9] = Forum.objects.get(id=15)
48 self.forums[10] = Forum.objects.get(slug='for-sale-trade')
49 self.forums[11] = Forum.objects.get(slug='musicians-gigs-wanted')
50 self.forums[12] = Forum.objects.get(slug='surf-videos')
51 self.forums[13] = Forum.objects.get(slug='sg101-podcast')
52 self.forums[14] = Forum.objects.get(slug='gigs')
53 self.forums[15] = Forum.objects.get(slug='music-reviews')
54 self.forums[18] = Forum.objects.get(slug='best-sg101')
55 except Forum.DoesNotExist:
56 sys.exit("Forum does not exist; check forum mapping.")
57
58 try:
59 with open(filename, "rb") as f:
60 self.reader = csv.DictReader(f)
61 num_rows = 0
62 try:
63 for row in self.reader:
64 self.process_row(row)
65 num_rows += 1
66 if self.show_progress and num_rows % 20 == 0:
67 sys.stdout.write('.')
68 sys.stdout.flush()
69 except csv.Error, e:
70 raise CommandError("CSV error: %s %s %s" % (
71 filename, self.reader.line_num, e))
72
73 print
74
75 except IOError:
76 raise CommandError("Could not open file: %s" % filename)
77
78 def process_row(self, row):
79 """
80 Process one row from the CSV file: create a Story object for
81 the row and save it in the database.
82
83 """
84 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
85
86 if row['topic_moved_id'] != '0':
87 return
88
89 try:
90 user = User.objects.get(id=int(row['topic_poster']))
91 except User.DoesNotExist:
92 print "Could not find user %s for topic %s; skipping." % (
93 row['topic_poster'], row['topic_id'])
94 return
95
96 creation_date = datetime.fromtimestamp(float(row['topic_time']))
97
98 title = row['topic_title'].decode('latin-1', 'replace')
99
100 try:
101 forum = self.forums[int(row['forum_id'])]
102 except KeyError:
103 print 'skipping topic "%s"' % title
104 return
105
106 topic = Topic(id=int(row['topic_id']),
107 forum=forum,
108 name=unescape(title),
109 creation_date=creation_date,
110 user=user,
111 view_count=int(row['topic_views']),
112 sticky=(int(row['topic_type']) != 0),
113 locked=(int(row['topic_status']) != 0),
114 update_date=creation_date)
115
116 topic.save()
117