Zensursula und negative Verantwortungsattribution - Netzpolitik

Anhang 1 – Der Topsy-Crawler
Anhang 1 Der Topsy-Crawler
# -*- coding: utf-8 -*-
#!/usr/bin/python
### Crawler for topsy
### Keven Richly and Lutz Mache, 2012
import datetime
import json
import sqlite3
import threading
import time
import urllib2
from multiprocessing import Queue
### variables
print_response_info = True
print_request_info = True
create_new_table = False
request_window = 604800 * 10 # 604800 = 7 Tage
max_request = 950
db_path = '/…/main.sqlite'
table_name = 'z_original_topsy_data'
### request parameters
tstart = 1217541600 #1. August 2008
tend = 1325286000 # 31. Dezember 2011
searchterm = 'zensursula'
type = 'tweet'
page = 1
perpage = 100
### classes
class Crawler(threading.Thread):
def __init__(self, t1, t2, results):
self.t1 = t1
self.t2 = t2
self.results = results
threading.Thread.__init__(self)
# returns the request url corresponding to the defined parameters
def get_request(self, start, end, term, page, type, perpage):
return 'http://otter.topsy.com/search.json?q=' + searchterm + '&sort_method=date&perpage='
+ str(perpage) + '&page=' + str(page) + '&mintime=' + str(start) + '&maxtime='
+ str(end) + '&type=' + type
class Page_Crawler(Crawler):
def __init__(self, t1, t2, current_page, count, results, errors, window):
self.errors = errors
self.page = current_page
self.count = count
self.window = window
Crawler.__init__(self, t1, t2, results)
# executes the given query and catches exceptions
def execute_query(self, query):
try:
return json.load(urllib2.urlopen(query))
except Exception, e:
print 'Error: query processing:'
print e
print 'actual time window was added to error list: '
print 'min: ' + str(self.t1)
print 'max: ' + str(self.t2)
self.errors.put([self.t1, self.t2, self.page])
def run(self):
topsy_response = self.execute_query(get_request(self.t1, self.t2, searchterm,
self.page, type, perpage))
items_count = len(topsy_response['response']['list'])
if (items_count > 0):
self.results.put(topsy_response['response']['list'])
self.count.put(items_count)
class Timewindow_Crawler(Crawler):
105/148