|
bgneal@294
|
1 """
|
|
bgneal@294
|
2 import_old_topics.py - For importing forum topics (threads) from SG101 1.0 as
|
|
bgneal@294
|
3 csv files.
|
|
bgneal@294
|
4
|
|
bgneal@294
|
5 """
|
|
bgneal@294
|
6 from __future__ import with_statement
|
|
bgneal@294
|
7 import csv
|
|
bgneal@294
|
8 import optparse
|
|
bgneal@294
|
9 import sys
|
|
bgneal@294
|
10 from datetime import datetime
|
|
bgneal@294
|
11
|
|
bgneal@294
|
12 from django.core.management.base import LabelCommand, CommandError
|
|
bgneal@294
|
13 from django.contrib.auth.models import User
|
|
bgneal@294
|
14
|
|
bgneal@294
|
15 from forums.models import Forum, Topic
|
|
bgneal@294
|
16 from legacy.phpbb import unescape
|
|
bgneal@294
|
17
|
|
bgneal@294
|
18
|
|
bgneal@294
|
19 class Command(LabelCommand):
|
|
bgneal@294
|
20 args = '<filename filename ...>'
|
|
bgneal@294
|
21 help = 'Imports forum topics from the old database in CSV format'
|
|
bgneal@294
|
22 option_list = LabelCommand.option_list + (
|
|
bgneal@294
|
23 optparse.make_option("-p", "--progress", action="store_true",
|
|
bgneal@294
|
24 help="Output a . after every 20 topics to show progress"),
|
|
bgneal@294
|
25 )
|
|
bgneal@294
|
26
|
|
bgneal@294
|
27 def handle_label(self, filename, **options):
|
|
bgneal@294
|
28 """
|
|
bgneal@294
|
29 Process each line in the CSV file given by filename by
|
|
bgneal@294
|
30 creating a new topic.
|
|
bgneal@294
|
31
|
|
bgneal@294
|
32 """
|
|
bgneal@294
|
33 self.show_progress = options.get('progress')
|
|
bgneal@294
|
34 self.users = {}
|
|
bgneal@294
|
35
|
|
bgneal@294
|
36 # Create a mapping from the old database's forums to our
|
|
bgneal@294
|
37 # forums
|
|
bgneal@294
|
38 self.forums = {}
|
|
bgneal@294
|
39 try:
|
|
bgneal@294
|
40 self.forums[2] = Forum.objects.get(slug='suggestion-box')
|
|
bgneal@294
|
41 self.forums[3] = Forum.objects.get(slug='surf-music')
|
|
bgneal@294
|
42 self.forums[4] = Forum.objects.get(slug='surf-musician')
|
|
bgneal@294
|
43 self.forums[5] = Forum.objects.get(slug='gear')
|
|
bgneal@294
|
44 self.forums[6] = Forum.objects.get(slug='recording-corner')
|
|
bgneal@294
|
45 self.forums[7] = Forum.objects.get(slug='shallow-end')
|
|
bgneal@294
|
46 self.forums[8] = Forum.objects.get(slug='surfguitar101-website')
|
|
bgneal@294
|
47 self.forums[9] = Forum.objects.get(id=15)
|
|
bgneal@294
|
48 self.forums[10] = Forum.objects.get(slug='for-sale-trade')
|
|
bgneal@294
|
49 self.forums[11] = Forum.objects.get(slug='musicians-gigs-wanted')
|
|
bgneal@294
|
50 self.forums[12] = Forum.objects.get(slug='surf-videos')
|
|
bgneal@294
|
51 self.forums[13] = Forum.objects.get(slug='sg101-podcast')
|
|
bgneal@294
|
52 self.forums[14] = Forum.objects.get(slug='gigs')
|
|
bgneal@294
|
53 self.forums[15] = Forum.objects.get(slug='music-reviews')
|
|
bgneal@294
|
54 self.forums[18] = Forum.objects.get(slug='best-sg101')
|
|
bgneal@294
|
55 except Forum.DoesNotExist:
|
|
bgneal@294
|
56 sys.exit("Forum does not exist; check forum mapping.")
|
|
bgneal@294
|
57
|
|
bgneal@294
|
58 try:
|
|
bgneal@294
|
59 with open(filename, "rb") as f:
|
|
bgneal@294
|
60 self.reader = csv.DictReader(f)
|
|
bgneal@294
|
61 num_rows = 0
|
|
bgneal@294
|
62 try:
|
|
bgneal@294
|
63 for row in self.reader:
|
|
bgneal@294
|
64 self.process_row(row)
|
|
bgneal@294
|
65 num_rows += 1
|
|
bgneal@294
|
66 if self.show_progress and num_rows % 20 == 0:
|
|
bgneal@294
|
67 sys.stdout.write('.')
|
|
bgneal@294
|
68 sys.stdout.flush()
|
|
bgneal@294
|
69 except csv.Error, e:
|
|
bgneal@294
|
70 raise CommandError("CSV error: %s %s %s" % (
|
|
bgneal@294
|
71 filename, self.reader.line_num, e))
|
|
bgneal@294
|
72
|
|
bgneal@294
|
73 print
|
|
bgneal@294
|
74
|
|
bgneal@294
|
75 except IOError:
|
|
bgneal@294
|
76 raise CommandError("Could not open file: %s" % filename)
|
|
bgneal@294
|
77
|
|
bgneal@294
|
78 def process_row(self, row):
|
|
bgneal@294
|
79 """
|
|
bgneal@294
|
80 Process one row from the CSV file: create a Story object for
|
|
bgneal@294
|
81 the row and save it in the database.
|
|
bgneal@294
|
82
|
|
bgneal@294
|
83 """
|
|
bgneal@294
|
84 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
|
|
bgneal@294
|
85
|
|
bgneal@294
|
86 if row['topic_moved_id'] != '0':
|
|
bgneal@294
|
87 return
|
|
bgneal@294
|
88
|
|
bgneal@294
|
89 try:
|
|
bgneal@294
|
90 user = User.objects.get(id=int(row['topic_poster']))
|
|
bgneal@294
|
91 except User.DoesNotExist:
|
|
bgneal@294
|
92 print "Could not find user %s for topic %s; skipping." % (
|
|
bgneal@294
|
93 row['topic_poster'], row['topic_id'])
|
|
bgneal@294
|
94 return
|
|
bgneal@294
|
95
|
|
bgneal@294
|
96 creation_date = datetime.fromtimestamp(float(row['topic_time']))
|
|
bgneal@294
|
97
|
|
bgneal@294
|
98 title = row['topic_title'].decode('latin-1', 'replace')
|
|
bgneal@294
|
99
|
|
bgneal@294
|
100 try:
|
|
bgneal@294
|
101 forum = self.forums[int(row['forum_id'])]
|
|
bgneal@294
|
102 except KeyError:
|
|
bgneal@294
|
103 print 'skipping topic "%s"' % title
|
|
bgneal@294
|
104 return
|
|
bgneal@294
|
105
|
|
bgneal@294
|
106 topic = Topic(id=int(row['topic_id']),
|
|
bgneal@294
|
107 forum=forum,
|
|
bgneal@294
|
108 name=unescape(title),
|
|
bgneal@294
|
109 creation_date=creation_date,
|
|
bgneal@294
|
110 user=user,
|
|
bgneal@294
|
111 view_count=int(row['topic_views']),
|
|
bgneal@294
|
112 sticky=(int(row['topic_type']) != 0),
|
|
bgneal@294
|
113 locked=(int(row['topic_status']) != 0),
|
|
bgneal@294
|
114 update_date=creation_date)
|
|
bgneal@294
|
115
|
|
bgneal@294
|
116 topic.save()
|
|
bgneal@294
|
117
|
