Mercurial > public > sg101

comparison gpp/legacy/management/commands/import_old_downloads.py @ 412:639cfdf59167

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Created import scripts for downloads and download comments.
author Brian Neal <bgneal@gmail.com>
date Thu, 07 Apr 2011 00:59:10 +0000
parents
children
comparison
equal deleted inserted replaced
411:97a426a67417 412:639cfdf59167
1 """
2 import_old_downloads.py - For importing downloads from SG101 1.0 as csv files.
3 """
4 from __future__ import with_statement
5 import csv
6 import datetime
7
8 from django.core.management.base import LabelCommand, CommandError
9 from django.contrib.auth.models import User
10
11 from downloads.models import Download, Category
12 from legacy.html2md import MarkdownWriter
13
14
15 # downloads with these lid's will be excluded
16 EXCLUDE_SET = set([1, 2, 3, 4, 277])
17
18 # Mapping of old category IDs to new; None means we don't plan on importing
19 CAT_MAP = {
20 4: None, # Misc
21 3: None, # Music
22 1: None, # Demos
23 6: 2, # Gear Samples
24 8: 4, # Ringtones
25 9: 8, # Tablature
26 10: 6, # Interviews
27 11: None, # 2008 MP3 Comp
28 12: 1, # Backing Tracks
29 13: None, # 2009 MP3 Comp
30 }
31
32 SG101_PREFIX = 'http://surfguitar101.com/'
33
34
35 class Command(LabelCommand):
36 args = '<filename filename ...>'
37 help = 'Imports downloads from the old database in CSV format'
38 md_writer = MarkdownWriter()
39
40 def handle_label(self, filename, **options):
41 """
42 Process each line in the CSV file given by filename by
43 creating a new object and saving it to the database.
44
45 """
46 self.cats = {}
47 try:
48 self.default_user = User.objects.get(pk=2)
49 except User.DoesNotExist:
50 raise CommandError("Need a default user with pk=2")
51
52 try:
53 with open(filename, "rb") as f:
54 self.reader = csv.DictReader(f)
55 try:
56 for row in self.reader:
57 self.process_row(row)
58 except csv.Error, e:
59 raise CommandError("CSV error: %s %s %s" % (
60 filename, self.reader.line_num, e))
61
62 except IOError:
63 raise CommandError("Could not open file: %s" % filename)
64
65 def get_category(self, old_cat_id):
66 """
67 Return the Category object for the row.
68
69 """
70 cat_id = CAT_MAP[old_cat_id]
71 if cat_id not in self.cats:
72 try:
73 cat = Category.objects.get(pk=cat_id)
74 except Category.DoesNotExist:
75 raise CommandError("Category does not exist: %s on line %s" % (
76 cat_id, self.reader.line_num))
77 else:
78 self.cats[cat_id] = cat
79 return self.cats[cat_id]
80
81 def get_user(self, username):
82 """
83 Return the user object for the given username.
84 If the user cannot be found, self.default_user is returned.
85
86 """
87 try:
88 return User.objects.get(username=username)
89 except User.DoesNotExist:
90 return self.default_user
91
92 def process_row(self, row):
93 """
94 Process one row from the CSV file: create an object for the row
95 and save it in the database.
96
97 """
98 lid = int(row['lid'])
99 if lid in EXCLUDE_SET:
100 return # skip
101
102 cat = int(row['cid'])
103 if CAT_MAP.get(cat) is None:
104 return # skip this one; we aren't carrying these over
105
106 dl_date = datetime.datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S")
107 old_url = row['url'].decode('latin-1')
108 if old_url.startswith(SG101_PREFIX):
109 old_url = old_url[len(SG101_PREFIX):]
110 if old_url.startswith('dls/'):
111 old_url = old_url[4:]
112 new_url = u'downloads/1.0/%s' % old_url
113
114 dl = Download(
115 id=lid,
116 title=row['title'].decode('latin-1'),
117 category=self.get_category(cat),
118 description=self.to_markdown(row['description'].decode('latin-1')),
119 file=new_url,
120 user=self.get_user(row['submitter']),
121 date_added=dl_date,
122 ip_address='127.0.0.1', # not available
123 hits=int(row['hits']),
124 average_score=float(row['downloadratingsummary']) / 2.0,
125 total_votes=int(row['totalvotes']),
126 is_public=True)
127 dl.save()
128 #print "cp %s %s" % (old_url, '/home/var/django-sites/sg101/sg101-trunk/media/' + new_url)
129
130 def to_markdown(self, s):
131 self.md_writer.reset()
132 self.md_writer.feed(s)
133 return self.md_writer.markdown()