Mercurial > public > sg101

annotate gpp/legacy/management/commands/import_old_news.py @ 294:254db4cb6a86

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Changes / scripts to import forums. Other tweaks and moving other import scripts to the legacy application.
author Brian Neal <bgneal@gmail.com>
date Wed, 05 Jan 2011 04:09:35 +0000
parents 2367c4795c92
children 4021ea1045f7
rev   line source
bgneal@291 1 """
bgneal@291 2 import_old_news.py - For importing news stories from SG101 1.0 as csv files.
bgneal@291 3 """
bgneal@291 4 from __future__ import with_statement
bgneal@291 5 import csv
bgneal@291 6 import optparse
bgneal@291 7 import sys
bgneal@291 8 from datetime import datetime
bgneal@291 9
bgneal@291 10 from django.core.management.base import LabelCommand, CommandError
bgneal@291 11 from django.contrib.auth.models import User
bgneal@291 12
bgneal@291 13 from news.models import Category, Story
bgneal@291 14 from legacy.phpbb import unescape
bgneal@292 15 import legacy.data
bgneal@291 16
bgneal@291 17
bgneal@291 18 class Command(LabelCommand):
bgneal@291 19 args = '<filename filename ...>'
bgneal@291 20 help = 'Imports news stories from the old database in CSV format'
bgneal@291 21 option_list = LabelCommand.option_list + (
bgneal@291 22 optparse.make_option("-p", "--progress", action="store_true",
bgneal@291 23 help="Output a . after every 20 stories to show progress"),
bgneal@291 24 )
bgneal@291 25
bgneal@291 26 def handle_label(self, filename, **options):
bgneal@291 27 """
bgneal@291 28 Process each line in the CSV file given by filename by
bgneal@291 29 creating a new story.
bgneal@291 30
bgneal@291 31 """
bgneal@291 32 self.show_progress = options.get('progress')
bgneal@291 33 self.users = {}
bgneal@291 34
bgneal@291 35 # Create a mapping from the old database's topics to our
bgneal@291 36 # Categories.
bgneal@291 37 self.topics = {}
bgneal@291 38 try:
bgneal@291 39 self.topics[2] = Category.objects.get(slug='site-news')
bgneal@291 40 self.topics[3] = Category.objects.get(slug='bands')
bgneal@291 41 self.topics[4] = Category.objects.get(slug='show-announcements')
bgneal@291 42 self.topics[5] = Category.objects.get(slug='show-reports')
bgneal@291 43 self.topics[6] = Category.objects.get(slug='gear')
bgneal@291 44 self.topics[7] = Category.objects.get(slug='reviews')
bgneal@291 45 self.topics[8] = Category.objects.get(slug='surf-scene-news')
bgneal@291 46 self.topics[9] = Category.objects.get(slug='articles')
bgneal@291 47 self.topics[10] = Category.objects.get(slug='interviews')
bgneal@291 48 self.topics[11] = Category.objects.get(slug='tablature')
bgneal@291 49 self.topics[12] = Category.objects.get(slug='featured-videos')
bgneal@291 50 except Category.DoesNotExist:
bgneal@291 51 sys.exit("Category does not exist; check topic mapping.")
bgneal@291 52
bgneal@291 53 try:
bgneal@291 54 with open(filename, "rb") as f:
bgneal@291 55 self.reader = csv.DictReader(f)
bgneal@291 56 num_rows = 0
bgneal@291 57 try:
bgneal@291 58 for row in self.reader:
bgneal@291 59 self.process_row(row)
bgneal@291 60 num_rows += 1
bgneal@291 61 if self.show_progress and num_rows % 20 == 0:
bgneal@291 62 sys.stdout.write('.')
bgneal@291 63 sys.stdout.flush()
bgneal@291 64 except csv.Error, e:
bgneal@291 65 raise CommandError("CSV error: %s %s %s" % (
bgneal@291 66 filename, self.reader.line_num, e))
bgneal@291 67
bgneal@291 68 print
bgneal@291 69
bgneal@291 70 except IOError:
bgneal@291 71 raise CommandError("Could not open file: %s" % filename)
bgneal@291 72
bgneal@291 73 def process_row(self, row):
bgneal@291 74 """
bgneal@291 75 Process one row from the CSV file: create a Story object for
bgneal@291 76 the row and save it in the database.
bgneal@291 77
bgneal@291 78 """
bgneal@291 79 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
bgneal@291 80
bgneal@291 81 try:
bgneal@291 82 submitter = self._get_user(row['informant'])
bgneal@291 83 except User.DoesNotExist:
bgneal@291 84 print "Could not find user %s for story %s; skipping." % (
bgneal@291 85 row['informant'], row['sid'])
bgneal@291 86 return
bgneal@291 87
bgneal@291 88 story = Story(id=int(row['sid']),
bgneal@294 89 title=unescape(row['title'].decode('latin-1')),
bgneal@291 90 submitter=submitter,
bgneal@291 91 category=self.topics[int(row['topic'])],
bgneal@294 92 short_text=row['hometext'].decode('latin-1'),
bgneal@294 93 long_text=row['bodytext'].decode('latin-1'),
bgneal@291 94 date_submitted=datetime.strptime(row['time'], "%Y-%m-%d %H:%M:%S"),
bgneal@291 95 allow_comments=True)
bgneal@291 96
bgneal@291 97 story.save()
bgneal@291 98
bgneal@291 99 def _get_user(self, username):
bgneal@291 100 """
bgneal@291 101 Returns the user object with the given username.
bgneal@291 102 Throws User.DoesNotExist if not found.
bgneal@291 103
bgneal@291 104 """
bgneal@291 105 try:
bgneal@291 106 return self.users[username]
bgneal@291 107 except KeyError:
bgneal@291 108 pass
bgneal@291 109
bgneal@291 110 try:
bgneal@291 111 user = User.objects.get(username=username)
bgneal@291 112 except User.DoesNotExist:
bgneal@291 113 try:
bgneal@292 114 user = User.objects.get(
bgneal@292 115 username=legacy.data.KNOWN_USERNAME_CHANGES[username])
bgneal@291 116 except KeyError:
bgneal@291 117 raise User.DoesNotExist
bgneal@291 118
bgneal@291 119 self.users[username] = user
bgneal@291 120 return user