changeset 334:6805d15cda13

Adding a script I had to write on the fly to filter out posts from the posts csv file that had no parent topics. MyISAM let me get away with that, but InnoDB won't.
author Brian Neal <bgneal@gmail.com>
date Sat, 26 Feb 2011 01:28:22 +0000 (2011-02-26)
parents 0bf5a5677067
children 46d7f69b61db
files tools/filter_posts.py
diffstat 1 files changed, 59 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/filter_posts.py	Sat Feb 26 01:28:22 2011 +0000
@@ -0,0 +1,59 @@
+"""
+filter_posts.py - A script to filter out posts that have no parent topic in the
+new database. 
+
+"""
+from __future__ import with_statement
+import csv
+import optparse
+import sys
+
+
+USAGE = "usage: %prog [options]"
+DESCRIPTION = """Filters out posts that have no parent topic."""
+
+
+def main(argv=None):
+
+    parser = optparse.OptionParser(usage=USAGE, description=DESCRIPTION)
+    parser.set_defaults(
+        topic_file='forums_topic.csv',
+        post_file='forums_post.csv',
+        output_file='forums_post_filtered.csv',
+    )
+    parser.add_option("-s", "--progress", action="store_true",
+        help="Output a . after every 100 posts to show progress [default: %default]")
+    parser.add_option("-t", "--topic-file",
+        help="Name of the topics csv file [default: %default]")
+    parser.add_option("-p", "--post-file",
+        help="Name of the posts csv file [default: %default]")
+    parser.add_option("-o", "--output-file",
+        help="Name of the output posts csv file [default: %default]")
+
+    opts, args = parser.parse_args(args=argv)
+
+    topics = set()
+    with open(opts.topic_file, "rb") as topic_file:
+        reader = csv.reader(topic_file)
+        for row in reader:
+            topics.add(int(row[0]))
+
+    with open(opts.post_file, "rb") as post_file:
+        reader = csv.reader(post_file)
+        # skip first row
+        print reader.next()
+        with open(opts.output_file, "wb") as output_file:
+            writer = csv.writer(output_file)
+            for row in reader:
+                topic = int(row[1])
+                if topic in topics:
+                    writer.writerow(row)
+                    
+
+if __name__ == '__main__':
+    try:
+        main()
+    except IOError, ex:
+        sys.exit("IO Error: %s" % ex)
+    except KeyboardInterrupt:
+        sys.exit("Control-C interrupt")