Mercurial > public > sg101

annotate legacy/management/commands/import_old_news.py @ 693:ad69236e8501

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
For issue #52, update many 3rd party Javascript libraries. Updated to jquery 1.10.2, jquery ui 1.10.3. This broke a lot of stuff. - Found a newer version of the jquery cycle all plugin (3.0.3). - Updated JPlayer to 2.4.0. - Updated to MarkItUp 1.1.14. This also required me to add multiline attributes set to true on various buttons in the markdown set. - As per a stackoverflow post, added some code to get multiline titles in a jQuery UI dialog. They removed that functionality but allow you to put it back. Tweaked the MarkItUp preview CSS to show blockquotes in italic. Did not update TinyMCE at this time. I'm not using the JQuery version and this version appears to work ok for now. What I should do is make a repo for MarkItUp and do a vendor branch thing so I don't have to futz around diffing directories to figure out if I'll lose changes when I update.
author Brian Neal <bgneal@gmail.com>
date Wed, 04 Sep 2013 19:55:20 -0500
parents ee87ea74d46b
children
rev   line source
bgneal@291 1 """
bgneal@291 2 import_old_news.py - For importing news stories from SG101 1.0 as csv files.
bgneal@291 3 """
bgneal@291 4 from __future__ import with_statement
bgneal@291 5 import csv
bgneal@291 6 import optparse
bgneal@291 7 import sys
bgneal@291 8 from datetime import datetime
bgneal@291 9
bgneal@291 10 from django.core.management.base import LabelCommand, CommandError
bgneal@291 11 from django.contrib.auth.models import User
bgneal@291 12
bgneal@291 13 from news.models import Category, Story
bgneal@291 14 from legacy.phpbb import unescape
bgneal@292 15 import legacy.data
bgneal@291 16
bgneal@291 17
bgneal@291 18 class Command(LabelCommand):
bgneal@291 19 args = '<filename filename ...>'
bgneal@291 20 help = 'Imports news stories from the old database in CSV format'
bgneal@291 21 option_list = LabelCommand.option_list + (
bgneal@291 22 optparse.make_option("-p", "--progress", action="store_true",
bgneal@291 23 help="Output a . after every 20 stories to show progress"),
bgneal@291 24 )
bgneal@291 25
bgneal@291 26 def handle_label(self, filename, **options):
bgneal@291 27 """
bgneal@291 28 Process each line in the CSV file given by filename by
bgneal@291 29 creating a new story.
bgneal@291 30
bgneal@291 31 """
bgneal@291 32 self.show_progress = options.get('progress')
bgneal@291 33 self.users = {}
bgneal@291 34
bgneal@291 35 # Create a mapping from the old database's topics to our
bgneal@291 36 # Categories.
bgneal@291 37 self.topics = {}
bgneal@291 38 try:
bgneal@291 39 self.topics[2] = Category.objects.get(slug='site-news')
bgneal@291 40 self.topics[3] = Category.objects.get(slug='bands')
bgneal@291 41 self.topics[4] = Category.objects.get(slug='show-announcements')
bgneal@291 42 self.topics[5] = Category.objects.get(slug='show-reports')
bgneal@291 43 self.topics[6] = Category.objects.get(slug='gear')
bgneal@291 44 self.topics[7] = Category.objects.get(slug='reviews')
bgneal@291 45 self.topics[8] = Category.objects.get(slug='surf-scene-news')
bgneal@291 46 self.topics[9] = Category.objects.get(slug='articles')
bgneal@291 47 self.topics[10] = Category.objects.get(slug='interviews')
bgneal@291 48 self.topics[11] = Category.objects.get(slug='tablature')
bgneal@291 49 self.topics[12] = Category.objects.get(slug='featured-videos')
bgneal@291 50 except Category.DoesNotExist:
bgneal@291 51 sys.exit("Category does not exist; check topic mapping.")
bgneal@291 52
bgneal@291 53 try:
bgneal@291 54 with open(filename, "rb") as f:
bgneal@291 55 self.reader = csv.DictReader(f)
bgneal@291 56 num_rows = 0
bgneal@291 57 try:
bgneal@291 58 for row in self.reader:
bgneal@291 59 self.process_row(row)
bgneal@291 60 num_rows += 1
bgneal@291 61 if self.show_progress and num_rows % 20 == 0:
bgneal@291 62 sys.stdout.write('.')
bgneal@291 63 sys.stdout.flush()
bgneal@291 64 except csv.Error, e:
bgneal@291 65 raise CommandError("CSV error: %s %s %s" % (
bgneal@291 66 filename, self.reader.line_num, e))
bgneal@291 67
bgneal@291 68 print
bgneal@291 69
bgneal@291 70 except IOError:
bgneal@291 71 raise CommandError("Could not open file: %s" % filename)
bgneal@291 72
bgneal@291 73 def process_row(self, row):
bgneal@291 74 """
bgneal@291 75 Process one row from the CSV file: create a Story object for
bgneal@291 76 the row and save it in the database.
bgneal@291 77
bgneal@291 78 """
bgneal@291 79 row = dict((k, v if v != 'NULL' else '') for k, v in row.iteritems())
bgneal@291 80
bgneal@291 81 try:
bgneal@291 82 submitter = self._get_user(row['informant'])
bgneal@291 83 except User.DoesNotExist:
bgneal@291 84 print "Could not find user %s for story %s; skipping." % (
bgneal@291 85 row['informant'], row['sid'])
bgneal@291 86 return
bgneal@291 87
bgneal@291 88 story = Story(id=int(row['sid']),
bgneal@294 89 title=unescape(row['title'].decode('latin-1')),
bgneal@291 90 submitter=submitter,
bgneal@291 91 category=self.topics[int(row['topic'])],
bgneal@294 92 short_text=row['hometext'].decode('latin-1'),
bgneal@294 93 long_text=row['bodytext'].decode('latin-1'),
bgneal@291 94 date_submitted=datetime.strptime(row['time'], "%Y-%m-%d %H:%M:%S"),
bgneal@291 95 allow_comments=True)
bgneal@291 96
bgneal@291 97 story.save()
bgneal@291 98
bgneal@291 99 def _get_user(self, username):
bgneal@291 100 """
bgneal@291 101 Returns the user object with the given username.
bgneal@291 102 Throws User.DoesNotExist if not found.
bgneal@291 103
bgneal@291 104 """
bgneal@291 105 try:
bgneal@291 106 return self.users[username]
bgneal@291 107 except KeyError:
bgneal@291 108 pass
bgneal@291 109
bgneal@291 110 try:
bgneal@291 111 user = User.objects.get(username=username)
bgneal@291 112 except User.DoesNotExist:
bgneal@535 113 old_name = username.lower()
bgneal@291 114 try:
bgneal@292 115 user = User.objects.get(
bgneal@535 116 username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
bgneal@291 117 except KeyError:
bgneal@291 118 raise User.DoesNotExist
bgneal@291 119
bgneal@291 120 self.users[username] = user
bgneal@291 121 return user