Mercurial > public > sg101

comparison legacy/management/commands/import_old_news.py @ 581:ee87ea74d46b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For Django 1.4, rearranged project structure for new manage.py.
author Brian Neal <bgneal@gmail.com>
date Sat, 05 May 2012 17:10:48 -0500
parents gpp/legacy/management/commands/import_old_news.py@4021ea1045f7
children
comparison
equal deleted inserted replaced
580:c525f3e0b5d0 581:ee87ea74d46b
1 """
2 import_old_news.py - For importing news stories from SG101 1.0 as csv files.
3 """
4 from __future__ import with_statement
5 import csv
6 import optparse
7 import sys
8 from datetime import datetime
9
10 from django.core.management.base import LabelCommand, CommandError
11 from django.contrib.auth.models import User
12
13 from news.models import Category, Story
14 from legacy.phpbb import unescape
15 import legacy.data
16
17
18 class Command(LabelCommand):
19 args = '<filename filename ...>'
20 help = 'Imports news stories from the old database in CSV format'
21 option_list = LabelCommand.option_list + (
22 optparse.make_option("-p", "--progress", action="store_true",
23 help="Output a . after every 20 stories to show progress"),
24 )
25
26 def handle_label(self, filename, **options):
27 """
28 Process each line in the CSV file given by filename by
29 creating a new story.
30
31 """
32 self.show_progress = options.get('progress')
33 self.users = {}
34
35 # Create a mapping from the old database's topics to our
36 # Categories.
37 self.topics = {}
38 try:
39 self.topics[2] = Category.objects.get(slug='site-news')
40 self.topics[3] = Category.objects.get(slug='bands')
41 self.topics[4] = Category.objects.get(slug='show-announcements')
42 self.topics[5] = Category.objects.get(slug='show-reports')
43 self.topics[6] = Category.objects.get(slug='gear')
44 self.topics[7] = Category.objects.get(slug='reviews')
45 self.topics[8] = Category.objects.get(slug='surf-scene-news')
46 self.topics[9] = Category.objects.get(slug='articles')
47 self.topics[10] = Category.objects.get(slug='interviews')
48 self.topics[11] = Category.objects.get(slug='tablature')
49 self.topics[12] = Category.objects.get(slug='featured-videos')
50 except Category.DoesNotExist:
51 sys.exit("Category does not exist; check topic mapping.")
52
53 try:
54 with open(filename, "rb") as f:
55 self.reader = csv.DictReader(f)
56 num_rows = 0
57 try:
58 for row in self.reader:
59 self.process_row(row)
60 num_rows += 1
61 if self.show_progress and num_rows % 20 == 0:
62 sys.stdout.write('.')
63 sys.stdout.flush()
64 except csv.Error, e:
65 raise CommandError("CSV error: %s %s %s" % (
66 filename, self.reader.line_num, e))
67
68 print
69
70 except IOError:
71 raise CommandError("Could not open file: %s" % filename)
72
73 def process_row(self, row):
74 """
75 Process one row from the CSV file: create a Story object for
76 the row and save it in the database.
77
78 """
79 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
80
81 try:
82 submitter = self._get_user(row['informant'])
83 except User.DoesNotExist:
84 print "Could not find user %s for story %s; skipping." % (
85 row['informant'], row['sid'])
86 return
87
88 story = Story(id=int(row['sid']),
89 title=unescape(row['title'].decode('latin-1')),
90 submitter=submitter,
91 category=self.topics[int(row['topic'])],
92 short_text=row['hometext'].decode('latin-1'),
93 long_text=row['bodytext'].decode('latin-1'),
94 date_submitted=datetime.strptime(row['time'], "%Y-%m-%d %H:%M:%S"),
95 allow_comments=True)
96
97 story.save()
98
99 def _get_user(self, username):
100 """
101 Returns the user object with the given username.
102 Throws User.DoesNotExist if not found.
103
104 """
105 try:
106 return self.users[username]
107 except KeyError:
108 pass
109
110 try:
111 user = User.objects.get(username=username)
112 except User.DoesNotExist:
113 old_name = username.lower()
114 try:
115 user = User.objects.get(
116 username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
117 except KeyError:
118 raise User.DoesNotExist
119
120 self.users[username] = user
121 return user