Mercurial > public > sg101

annotate gpp/legacy/management/commands/import_old_news.py @ 291:a6357f2bcbbc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added a legacy command to import old news stories.
author Brian Neal <bgneal@gmail.com>
date Fri, 24 Dec 2010 20:45:33 +0000
parents
children 2367c4795c92
rev   line source
bgneal@291 1 """
bgneal@291 2 import_old_news.py - For importing news stories from SG101 1.0 as csv files.
bgneal@291 3 """
bgneal@291 4 from __future__ import with_statement
bgneal@291 5 import csv
bgneal@291 6 import optparse
bgneal@291 7 import sys
bgneal@291 8 from datetime import datetime
bgneal@291 9
bgneal@291 10 from django.core.management.base import LabelCommand, CommandError
bgneal@291 11 from django.contrib.auth.models import User
bgneal@291 12
bgneal@291 13 from news.models import Category, Story
bgneal@291 14 from legacy.phpbb import unescape
bgneal@291 15
bgneal@291 16 KNOWN_USERNAME_CHANGES = {
bgneal@291 17 'cavefishbutchdelux': 'butchdelux',
bgneal@291 18 'Findicator1': 'WaveOhhh',
bgneal@291 19 }
bgneal@291 20
bgneal@291 21 class Command(LabelCommand):
bgneal@291 22 args = '<filename filename ...>'
bgneal@291 23 help = 'Imports news stories from the old database in CSV format'
bgneal@291 24 option_list = LabelCommand.option_list + (
bgneal@291 25 optparse.make_option("-p", "--progress", action="store_true",
bgneal@291 26 help="Output a . after every 20 stories to show progress"),
bgneal@291 27 )
bgneal@291 28
bgneal@291 29 def handle_label(self, filename, **options):
bgneal@291 30 """
bgneal@291 31 Process each line in the CSV file given by filename by
bgneal@291 32 creating a new story.
bgneal@291 33
bgneal@291 34 """
bgneal@291 35 self.show_progress = options.get('progress')
bgneal@291 36 self.users = {}
bgneal@291 37
bgneal@291 38 # Create a mapping from the old database's topics to our
bgneal@291 39 # Categories.
bgneal@291 40 self.topics = {}
bgneal@291 41 try:
bgneal@291 42 self.topics[2] = Category.objects.get(slug='site-news')
bgneal@291 43 self.topics[3] = Category.objects.get(slug='bands')
bgneal@291 44 self.topics[4] = Category.objects.get(slug='show-announcements')
bgneal@291 45 self.topics[5] = Category.objects.get(slug='show-reports')
bgneal@291 46 self.topics[6] = Category.objects.get(slug='gear')
bgneal@291 47 self.topics[7] = Category.objects.get(slug='reviews')
bgneal@291 48 self.topics[8] = Category.objects.get(slug='surf-scene-news')
bgneal@291 49 self.topics[9] = Category.objects.get(slug='articles')
bgneal@291 50 self.topics[10] = Category.objects.get(slug='interviews')
bgneal@291 51 self.topics[11] = Category.objects.get(slug='tablature')
bgneal@291 52 self.topics[12] = Category.objects.get(slug='featured-videos')
bgneal@291 53 except Category.DoesNotExist:
bgneal@291 54 sys.exit("Category does not exist; check topic mapping.")
bgneal@291 55
bgneal@291 56 try:
bgneal@291 57 with open(filename, "rb") as f:
bgneal@291 58 self.reader = csv.DictReader(f)
bgneal@291 59 num_rows = 0
bgneal@291 60 try:
bgneal@291 61 for row in self.reader:
bgneal@291 62 self.process_row(row)
bgneal@291 63 num_rows += 1
bgneal@291 64 if self.show_progress and num_rows % 20 == 0:
bgneal@291 65 sys.stdout.write('.')
bgneal@291 66 sys.stdout.flush()
bgneal@291 67 except csv.Error, e:
bgneal@291 68 raise CommandError("CSV error: %s %s %s" % (
bgneal@291 69 filename, self.reader.line_num, e))
bgneal@291 70
bgneal@291 71 print
bgneal@291 72
bgneal@291 73 except IOError:
bgneal@291 74 raise CommandError("Could not open file: %s" % filename)
bgneal@291 75
bgneal@291 76 def process_row(self, row):
bgneal@291 77 """
bgneal@291 78 Process one row from the CSV file: create a Story object for
bgneal@291 79 the row and save it in the database.
bgneal@291 80
bgneal@291 81 """
bgneal@291 82 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
bgneal@291 83
bgneal@291 84 try:
bgneal@291 85 submitter = self._get_user(row['informant'])
bgneal@291 86 except User.DoesNotExist:
bgneal@291 87 print "Could not find user %s for story %s; skipping." % (
bgneal@291 88 row['informant'], row['sid'])
bgneal@291 89 return
bgneal@291 90
bgneal@291 91 story = Story(id=int(row['sid']),
bgneal@291 92 title=unescape(row['title']),
bgneal@291 93 submitter=submitter,
bgneal@291 94 category=self.topics[int(row['topic'])],
bgneal@291 95 short_text=row['hometext'],
bgneal@291 96 long_text=row['bodytext'],
bgneal@291 97 date_submitted=datetime.strptime(row['time'], "%Y-%m-%d %H:%M:%S"),
bgneal@291 98 allow_comments=True)
bgneal@291 99
bgneal@291 100 story.save()
bgneal@291 101
bgneal@291 102 def _get_user(self, username):
bgneal@291 103 """
bgneal@291 104 Returns the user object with the given username.
bgneal@291 105 Throws User.DoesNotExist if not found.
bgneal@291 106
bgneal@291 107 """
bgneal@291 108 try:
bgneal@291 109 return self.users[username]
bgneal@291 110 except KeyError:
bgneal@291 111 pass
bgneal@291 112
bgneal@291 113 try:
bgneal@291 114 user = User.objects.get(username=username)
bgneal@291 115 except User.DoesNotExist:
bgneal@291 116 try:
bgneal@291 117 user = User.objects.get(username=KNOWN_USERNAME_CHANGES[username])
bgneal@291 118 except KeyError:
bgneal@291 119 raise User.DoesNotExist
bgneal@291 120
bgneal@291 121 self.users[username] = user
bgneal@291 122 return user