Zensursula und negative Verantwortungsattribution - Netzpolitik

Anhang 1 – Der Topsy-Crawler
integer, trackback_date text, trackback_author_nick text, url text, title text, firstpost_date
text, content text, mytype text, score real, topsy_author_img text, highlight text,
trackback_author_url text, trackback_permalink text, topsy_author_url text,
trackback_author_name text, topsy_trackback_url text, searchterm text,
firstpost_date_timestamp text, PRIMARY KEY (trackback_date, url))'
c.execute(sql_query)
except sqlite3.OperationalError, e:
table_name = table_name + '_' + str(time.time()).replace('.','')
create_new_db_table()
# inserts a tupel into the database
def insert_db(i):
insert(i['hits'], i['trackback_total'], i['trackback_date'],
i['trackback_author_nick'], i['url'], i['title'], i['firstpost_date'], i['content'],
i['mytype'], i['score'], i['topsy_author_img'], i['highlight'], i['trackback_author_url'],
i['trackback_permalink'], i['topsy_author_url'], i['trackback_author_name'], searchterm,
i['topsy_trackback_url'])
# executes the insert command
def insert(hits, trackback_total, trackback_date, trackback_author_nick, url, title,
firstpost_date, content, mytype, score, topsy_author_img, highlight, trackback_author_url,
trackback_permalink, topsy_author_url, trackback_author_name, searchterm,
topsy_trackback_url):
sql_query = 'insert into ' + table_name + ' values
(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
c.execute(sql_query, [hits, trackback_total, trackback_date, trackback_author_nick,
url, title, firstpost_date, content, mytype, score, topsy_author_img, highlight,
trackback_author_url, trackback_permalink, topsy_author_url, trackback_author_name,
topsy_trackback_url, searchterm, to_timestamp(firstpost_date)])
conn.commit()
### main
# opens a database connection
conn = sqlite3.connect(db_path)
c = conn.cursor()
if print_request_info:
topsy_response = execute_query(get_request_limit())
print_request_limit_information(topsy_response)
if print_response_info:
topsy_response = execute_query(get_request(tstart, tend, searchterm, page, type,
perpage))
print_response_information(topsy_response)
if create_new_table:
create_new_db_table()
print 'Created Table: ' + table_name
# calculates the number of time windows
window_count = 1
window_number = ((tend - tstart) / request_window) + 1
print 'Window number: ' + str(window_number)
results = Queue()
threads = []
t1, t2 = set_window(tstart)
# processes all time time windows
while(t2 < tend):
thread = Timewindow_Crawler(t1, t2, window_count, results)
threads.append(thread)
thread.start()
t1, t2 = set_window(t2)
window_count += 1
# processes the last time window
thread = Timewindow_Crawler(t1, tend, window_count, results)
threads.append(thread)
thread.start()
# wait for all threads
for t in threads:
t.join()
write_to_database(results)
c.close
conn.close()
108/148