Mercurial > public > sg101

comparison gpp/legacy/management/commands/import_old_topics.py @ 294:254db4cb6a86

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Changes / scripts to import forums. Other tweaks and moving other import scripts to the legacy application.
author Brian Neal <bgneal@gmail.com>
date Wed, 05 Jan 2011 04:09:35 +0000
parents
children
comparison
equal deleted inserted replaced
293:c92fb89dbc7d 294:254db4cb6a86
1 """
2 import_old_topics.py - For importing forum topics (threads) from SG101 1.0 as
3 csv files.
4
5 """
6 from __future__ import with_statement
7 import csv
8 import optparse
9 import sys
10 from datetime import datetime
11
12 from django.core.management.base import LabelCommand, CommandError
13 from django.contrib.auth.models import User
14
15 from forums.models import Forum, Topic
16 from legacy.phpbb import unescape
17
18
19 class Command(LabelCommand):
20 args = '<filename filename ...>'
21 help = 'Imports forum topics from the old database in CSV format'
22 option_list = LabelCommand.option_list + (
23 optparse.make_option("-p", "--progress", action="store_true",
24 help="Output a . after every 20 topics to show progress"),
25 )
26
27 def handle_label(self, filename, **options):
28 """
29 Process each line in the CSV file given by filename by
30 creating a new topic.
31
32 """
33 self.show_progress = options.get('progress')
34 self.users = {}
35
36 # Create a mapping from the old database's forums to our
37 # forums
38 self.forums = {}
39 try:
40 self.forums[2] = Forum.objects.get(slug='suggestion-box')
41 self.forums[3] = Forum.objects.get(slug='surf-music')
42 self.forums[4] = Forum.objects.get(slug='surf-musician')
43 self.forums[5] = Forum.objects.get(slug='gear')
44 self.forums[6] = Forum.objects.get(slug='recording-corner')
45 self.forums[7] = Forum.objects.get(slug='shallow-end')
46 self.forums[8] = Forum.objects.get(slug='surfguitar101-website')
47 self.forums[9] = Forum.objects.get(id=15)
48 self.forums[10] = Forum.objects.get(slug='for-sale-trade')
49 self.forums[11] = Forum.objects.get(slug='musicians-gigs-wanted')
50 self.forums[12] = Forum.objects.get(slug='surf-videos')
51 self.forums[13] = Forum.objects.get(slug='sg101-podcast')
52 self.forums[14] = Forum.objects.get(slug='gigs')
53 self.forums[15] = Forum.objects.get(slug='music-reviews')
54 self.forums[18] = Forum.objects.get(slug='best-sg101')
55 except Forum.DoesNotExist:
56 sys.exit("Forum does not exist; check forum mapping.")
57
58 try:
59 with open(filename, "rb") as f:
60 self.reader = csv.DictReader(f)
61 num_rows = 0
62 try:
63 for row in self.reader:
64 self.process_row(row)
65 num_rows += 1
66 if self.show_progress and num_rows % 20 == 0:
67 sys.stdout.write('.')
68 sys.stdout.flush()
69 except csv.Error, e:
70 raise CommandError("CSV error: %s %s %s" % (
71 filename, self.reader.line_num, e))
72
73 print
74
75 except IOError:
76 raise CommandError("Could not open file: %s" % filename)
77
78 def process_row(self, row):
79 """
80 Process one row from the CSV file: create a Story object for
81 the row and save it in the database.
82
83 """
84 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
85
86 if row['topic_moved_id'] != '0':
87 return
88
89 try:
90 user = User.objects.get(id=int(row['topic_poster']))
91 except User.DoesNotExist:
92 print "Could not find user %s for topic %s; skipping." % (
93 row['topic_poster'], row['topic_id'])
94 return
95
96 creation_date = datetime.fromtimestamp(float(row['topic_time']))
97
98 title = row['topic_title'].decode('latin-1', 'replace')
99
100 try:
101 forum = self.forums[int(row['forum_id'])]
102 except KeyError:
103 print 'skipping topic "%s"' % title
104 return
105
106 topic = Topic(id=int(row['topic_id']),
107 forum=forum,
108 name=unescape(title),
109 creation_date=creation_date,
110 user=user,
111 view_count=int(row['topic_views']),
112 sticky=(int(row['topic_type']) != 0),
113 locked=(int(row['topic_status']) != 0),
114 update_date=creation_date)
115
116 topic.save()
117