Mercurial > public > sg101
changeset 412:639cfdf59167
Created import scripts for downloads and download comments.
| author | Brian Neal <bgneal@gmail.com> |
|---|---|
| date | Thu, 07 Apr 2011 00:59:10 +0000 |
| parents | 97a426a67417 |
| children | 6144023ebea8 |
| files | gpp/downloads/models.py gpp/legacy/management/commands/import_old_download_comments.py gpp/legacy/management/commands/import_old_downloads.py gpp/legacy/management/commands/import_old_links.py |
| diffstat | 4 files changed, 226 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- a/gpp/downloads/models.py Sat Apr 02 01:17:19 2011 +0000 +++ b/gpp/downloads/models.py Thu Apr 07 00:59:10 2011 +0000 @@ -158,8 +158,8 @@ def __unicode__(self): return u"%s voted on '%s' on %s" % ( - self.user.username, - self.download.title, + self.user.username, + self.download.title, self.vote_date.strftime('%b %d, %Y %H:%M:%S')) class Meta:
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gpp/legacy/management/commands/import_old_download_comments.py Thu Apr 07 00:59:10 2011 +0000 @@ -0,0 +1,90 @@ +""" +import_old_download_comments.py - For importing download comments from SG101 1.0 +as csv files. + +""" +from __future__ import with_statement +import csv +from datetime import datetime + +from django.core.management.base import LabelCommand, CommandError +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType + +from downloads.models import Download, VoteRecord +from comments.models import Comment +from legacy.html2md import MarkdownWriter +import legacy.data + + +class Command(LabelCommand): + args = '<filename filename ...>' + help = 'Imports download comments from the old database in CSV format' + md_writer = MarkdownWriter() + + def handle_label(self, filename, **options): + """ + Process each line in the CSV file given by filename by + creating a new object and saving it to the database. + + """ + try: + with open(filename, "rb") as f: + self.reader = csv.DictReader(f) + try: + for row in self.reader: + self.process_row(row) + except csv.Error, e: + raise CommandError("CSV error: %s %s %s" % ( + filename, self.reader.line_num, e)) + + except IOError: + raise CommandError("Could not open file: %s" % filename) + + def process_row(self, row): + """ + Process one row from the CSV file: create an object for the row + and save it in the database. + + """ + dl_id = int(row['ratinglid']) + if dl_id in (1, 2, 3, 4): + return + + try: + dl = Download.objects.get(pk=dl_id) + except Download.DoesNotExist: + return + + try: + user = User.objects.get(username=row['ratinguser']) + except User.DoesNotExist: + try: + user = User.objects.get( + username=legacy.data.KNOWN_USERNAME_CHANGES[row['ratinguser']]) + except (User.DoesNotExist, KeyError): + return + + vote_date = datetime.strptime(row['ratingtimestamp'], "%Y-%m-%d %H:%M:%S") + + comment_text = row['ratingcomments'].decode('latin-1').strip() + if comment_text: + comment = Comment( + content_type=ContentType.objects.get_for_model(dl), + object_id=dl.id, + user=user, + comment=comment_text, + creation_date=vote_date, + ip_address = row['ratinghostname'], + is_public = True, + is_removed = False, + ) + comment.save() + + vr = VoteRecord(download=dl, user=user, vote_date=vote_date) + vr.save() + + def to_markdown(self, s): + self.md_writer.reset() + self.md_writer.feed(s) + return self.md_writer.markdown()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gpp/legacy/management/commands/import_old_downloads.py Thu Apr 07 00:59:10 2011 +0000 @@ -0,0 +1,133 @@ +""" +import_old_downloads.py - For importing downloads from SG101 1.0 as csv files. +""" +from __future__ import with_statement +import csv +import datetime + +from django.core.management.base import LabelCommand, CommandError +from django.contrib.auth.models import User + +from downloads.models import Download, Category +from legacy.html2md import MarkdownWriter + + +# downloads with these lid's will be excluded +EXCLUDE_SET = set([1, 2, 3, 4, 277]) + +# Mapping of old category IDs to new; None means we don't plan on importing +CAT_MAP = { + 4: None, # Misc + 3: None, # Music + 1: None, # Demos + 6: 2, # Gear Samples + 8: 4, # Ringtones + 9: 8, # Tablature + 10: 6, # Interviews + 11: None, # 2008 MP3 Comp + 12: 1, # Backing Tracks + 13: None, # 2009 MP3 Comp +} + +SG101_PREFIX = 'http://surfguitar101.com/' + + +class Command(LabelCommand): + args = '<filename filename ...>' + help = 'Imports downloads from the old database in CSV format' + md_writer = MarkdownWriter() + + def handle_label(self, filename, **options): + """ + Process each line in the CSV file given by filename by + creating a new object and saving it to the database. + + """ + self.cats = {} + try: + self.default_user = User.objects.get(pk=2) + except User.DoesNotExist: + raise CommandError("Need a default user with pk=2") + + try: + with open(filename, "rb") as f: + self.reader = csv.DictReader(f) + try: + for row in self.reader: + self.process_row(row) + except csv.Error, e: + raise CommandError("CSV error: %s %s %s" % ( + filename, self.reader.line_num, e)) + + except IOError: + raise CommandError("Could not open file: %s" % filename) + + def get_category(self, old_cat_id): + """ + Return the Category object for the row. + + """ + cat_id = CAT_MAP[old_cat_id] + if cat_id not in self.cats: + try: + cat = Category.objects.get(pk=cat_id) + except Category.DoesNotExist: + raise CommandError("Category does not exist: %s on line %s" % ( + cat_id, self.reader.line_num)) + else: + self.cats[cat_id] = cat + return self.cats[cat_id] + + def get_user(self, username): + """ + Return the user object for the given username. + If the user cannot be found, self.default_user is returned. + + """ + try: + return User.objects.get(username=username) + except User.DoesNotExist: + return self.default_user + + def process_row(self, row): + """ + Process one row from the CSV file: create an object for the row + and save it in the database. + + """ + lid = int(row['lid']) + if lid in EXCLUDE_SET: + return # skip + + cat = int(row['cid']) + if CAT_MAP.get(cat) is None: + return # skip this one; we aren't carrying these over + + dl_date = datetime.datetime.strptime(row['date'], "%Y-%m-%d %H:%M:%S") + old_url = row['url'].decode('latin-1') + if old_url.startswith(SG101_PREFIX): + old_url = old_url[len(SG101_PREFIX):] + if old_url.startswith('dls/'): + old_url = old_url[4:] + new_url = u'downloads/1.0/%s' % old_url + + dl = Download( + id=lid, + title=row['title'].decode('latin-1'), + category=self.get_category(cat), + description=self.to_markdown(row['description'].decode('latin-1')), + file=new_url, + user=self.get_user(row['submitter']), + date_added=dl_date, + ip_address='127.0.0.1', # not available + hits=int(row['hits']), + average_score=float(row['downloadratingsummary']) / 2.0, + total_votes=int(row['totalvotes']), + is_public=True) + dl.save() + #print "cp %s %s" % (old_url, '/home/var/django-sites/sg101/sg101-trunk/media/' + new_url) + + def to_markdown(self, s): + self.md_writer.reset() + self.md_writer.feed(s) + return self.md_writer.markdown()
--- a/gpp/legacy/management/commands/import_old_links.py Sat Apr 02 01:17:19 2011 +0000 +++ b/gpp/legacy/management/commands/import_old_links.py Thu Apr 07 00:59:10 2011 +0000 @@ -25,7 +25,7 @@ try: self.default_user = User.objects.get(pk=2) except User.DoesNotExist: - raise CommandError("Need a default user with pk=1") + raise CommandError("Need a default user with pk=2") try: with open(filename, "rb") as f: