Mercurial > public > sg101

annotate legacy/management/commands/import_old_potd.py @ 887:9a15f7c27526

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Actually save model object upon change. This commit was tested on the comments model. Additional logging added. Added check for Markdown image references. Added TODOs after observing behavior on comments.
author Brian Neal <bgneal@gmail.com>
date Tue, 03 Feb 2015 21:09:44 -0600
parents ee87ea74d46b
children
rev   line source
bgneal@535 1 """
bgneal@535 2 import_old_potd.py - For importing POTD's from SG101 1.0 as csv files.
bgneal@535 3
bgneal@535 4 """
bgneal@535 5 from __future__ import with_statement
bgneal@535 6 import csv
bgneal@535 7 import optparse
bgneal@535 8 import sys
bgneal@535 9 from datetime import datetime
bgneal@535 10
bgneal@535 11 from django.core.management.base import LabelCommand, CommandError
bgneal@535 12 from django.contrib.auth.models import User
bgneal@535 13
bgneal@535 14 from potd.models import Photo
bgneal@535 15 from legacy.phpbb import unescape
bgneal@535 16 import legacy.data
bgneal@535 17
bgneal@535 18
bgneal@535 19 ID_OFFSET = 100
bgneal@535 20
bgneal@535 21
bgneal@535 22 class PathError(Exception):
bgneal@535 23 pass
bgneal@535 24
bgneal@535 25 def convert_path(old_path):
bgneal@535 26 """
bgneal@535 27 Converts the old POTD path to a new one.
bgneal@535 28
bgneal@535 29 """
bgneal@535 30 if old_path.startswith('images/potd/'):
bgneal@535 31 return "potd/1.0/%s" % old_path[12:]
bgneal@535 32 else:
bgneal@535 33 raise PathError("Unknown path %s" % old_path)
bgneal@535 34
bgneal@535 35
bgneal@535 36 class Command(LabelCommand):
bgneal@535 37 args = '<filename filename ...>'
bgneal@535 38 help = "Imports POTD's from the old database in CSV format"
bgneal@535 39 option_list = LabelCommand.option_list + (
bgneal@535 40 optparse.make_option("-p", "--progress", action="store_true",
bgneal@535 41 help="Output a . after every 20 items to show progress"),
bgneal@535 42 )
bgneal@535 43
bgneal@535 44 def handle_label(self, filename, **options):
bgneal@535 45 """
bgneal@535 46 Process each line in the CSV file given by filename by
bgneal@535 47 creating a new Photo
bgneal@535 48
bgneal@535 49 """
bgneal@535 50 self.show_progress = options.get('progress')
bgneal@535 51 self.users = {}
bgneal@535 52
bgneal@535 53 try:
bgneal@535 54 with open(filename, "rb") as f:
bgneal@535 55 self.reader = csv.DictReader(f)
bgneal@535 56 num_rows = 0
bgneal@535 57 try:
bgneal@535 58 for row in self.reader:
bgneal@535 59 self.process_row(row)
bgneal@535 60 num_rows += 1
bgneal@535 61 if self.show_progress and num_rows % 20 == 0:
bgneal@535 62 sys.stdout.write('.')
bgneal@535 63 sys.stdout.flush()
bgneal@535 64 except csv.Error, e:
bgneal@535 65 raise CommandError("CSV error: %s %s %s" % (
bgneal@535 66 filename, self.reader.line_num, e))
bgneal@535 67
bgneal@535 68 print
bgneal@535 69
bgneal@535 70 except IOError:
bgneal@535 71 raise CommandError("Could not open file: %s" % filename)
bgneal@535 72
bgneal@535 73 def process_row(self, row):
bgneal@535 74 """
bgneal@535 75 Process one row from the CSV file: create a Photo object for
bgneal@535 76 the row and save it in the database.
bgneal@535 77
bgneal@535 78 """
bgneal@535 79 try:
bgneal@535 80 submitter = self._get_user(row['submitted_by'].decode('latin-1'))
bgneal@535 81 except User.DoesNotExist:
bgneal@535 82 print "Could not find user %s for potd %s; skipping." % (
bgneal@535 83 row['submitted_by'], row['pid'])
bgneal@535 84 return
bgneal@535 85
bgneal@535 86 desc = row['description'].decode('latin-1').replace('\n', '\n<br />')
bgneal@535 87
bgneal@535 88 try:
bgneal@535 89 photo = Photo(
bgneal@535 90 id=int(row['pid']) + ID_OFFSET,
bgneal@535 91 photo=convert_path(row['photo_path']),
bgneal@535 92 thumb=convert_path(row['thumb_path']),
bgneal@535 93 caption=unescape(row['title'].decode('latin-1')),
bgneal@535 94 description=desc,
bgneal@535 95 user=submitter,
bgneal@535 96 date_added=datetime.strptime(row['date_added'],
bgneal@535 97 "%Y-%m-%d %H:%M:%S"),
bgneal@535 98 potd_count=int(row['chosen_count']))
bgneal@535 99 except PathError, ex:
bgneal@535 100 self.stderr.write("\n%s, skipping\n" % ex)
bgneal@535 101 return
bgneal@535 102
bgneal@535 103 photo.save()
bgneal@535 104
bgneal@535 105 def _get_user(self, username):
bgneal@535 106 """
bgneal@535 107 Returns the user object with the given username.
bgneal@535 108 Throws User.DoesNotExist if not found.
bgneal@535 109
bgneal@535 110 """
bgneal@535 111 try:
bgneal@535 112 return self.users[username]
bgneal@535 113 except KeyError:
bgneal@535 114 pass
bgneal@535 115
bgneal@535 116 try:
bgneal@535 117 user = User.objects.get(username=username)
bgneal@535 118 except User.DoesNotExist:
bgneal@535 119 old_name = username.lower()
bgneal@535 120 try:
bgneal@535 121 user = User.objects.get(
bgneal@535 122 username=legacy.data.KNOWN_USERNAME_CHANGES[old_name])
bgneal@535 123 except KeyError:
bgneal@535 124 raise User.DoesNotExist
bgneal@535 125
bgneal@535 126 self.users[username] = user
bgneal@535 127 return user