Compare commits

...

2 commits

Author SHA1 Message Date
7cc3d1b7e4 fix preprocessing of tags 2023-11-01 19:42:22 +01:00
116cce79a4 update shebang for python3 2023-11-01 19:41:44 +01:00

View file

@ -1,4 +1,4 @@
#! python
#! python3
import locale
import logging
import os
@ -74,12 +74,15 @@ def download(id: int):
author = None
try:
category = pC.find("span", {"class": "categoryInfo"}).find().text
category = pC.find("span", {"class": "categoryInfo"}).find_all()
category = [c.text for c in category]
category = ";".join(category)
except AttributeError:
category = None
try:
tags = [x.text for x in pC.find("div", {"class": "tagsInfo"}).find_all("a")]
tags = ";".join(tags)
except AttributeError:
tags = None
@ -127,7 +130,6 @@ def run_downloads(min_id: int, max_id: int, num_threads: int = 8):
# sqlite can't handle lists so let's convert them to a single row csv
# TODO: make sure our database is properly normalized
df = pd.DataFrame(res)
df.tags = df.tags.apply(lambda x: "; ".join(x) if x is not None else None)
return df