Zensursula und negative Verantwortungsattribution - Netzpolitik

Anhang 1 – Der Topsy-Crawler
### methods
# returns the request url corresponding to the defined parameters
def get_request(start, end, term, page, type, perpage):
return 'http://otter.topsy.com/search.json?q=' + searchterm + '&sort_method=date&perpage='
+ str(perpage) + '&page=' + str(page) + '&mintime=' + str(start) + '&maxtime='
+ str(end) + '&type=' + type
# returns the request url for the request limit
def get_request_limit():
return 'http://otter.topsy.com/credit.json'
# prints information about the request and the response
def print_response_information(response):
print ''
print 'Request Information'
print 'url: ' + response['request']['url']
for key in response['request']['parameters']:
if (key == 'maxtime') or (key == 'mintime'):
print key + ': ' +
to_timestamp(float(response['request']['parameters'][key]))
else:
print key + ': ' + response['request']['parameters'][key]
print ''
print 'Response Information'
for key in response['response']:
if (key != 'list'):
print key + ': ' + str(response['response'][key])
print ''
# prints information about the request limit
def print_request_limit_information(response):
print 'Request Limit Information'
print 'last reset: ' + to_timestamp(response['response']['reset'])
print 'limit: ' + str(response['response']['limit'])
print 'refresh_in_secs: ' + str(response['response']['refresh_in_secs'])
print 'remaining requests: ' + str(response['response']['remaining'])
print ''
# executes the given query and catches exceptions
def execute_query(query):
try:
return json.load(urllib2.urlopen(query))
except Exception, e:
print 'Error: query processing:'
raise e
# set the limits of the time window
def set_window(time):
return time + 1, time + request_window
# returns timestamp of an epoche time value
def to_timestamp(date):
return time.strftime("%d.%m.%Y %H:%M:%S", time.gmtime(date))
# write values to database
def write_to_database(results):
print 'Writing to database'
insert_counter = 0
duplicate_counter = 0
while not results.empty():
result = results.get()
for item in result:
try:
insert_db(item)
insert_counter += 1
#print 'Inserting succesful'
except sqlite3.IntegrityError, e:
duplicate_counter += 1
print 'Finished writing'
print 'Insert ' + str(insert_counter) + ' new values'
print 'Crawled ' + str(duplicate_counter) + ' known values'
# creates a table in the database
def create_new_db_table():
global table_name
try:
sql_query = 'create table ' + table_name + '(hits integer, trackback_total
107/148