JuliusFreudenberger/tripadvisor-parser
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add multithreading per attraction

Browse source
This commit is contained in:
Julius Freudenberger 2023-05-01 11:57:33 +02:00
parent 6dbcf39484
commit d5081f6cd8
1 changed files with 7 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
main.py
View file

@ -1,19 +1,13 @@
from concurrent.futures import ThreadPoolExecutor
from csv_exporter import export_attraction from csv_exporter import export_attraction
from tripadvisor_attraction import TripadvisorAttraction
from tripadvisor_parser import TripadvisorAttractionParser from tripadvisor_parser import TripadvisorAttractionParser
def parse_locations(urls: [str]) -> [TripadvisorAttraction]: def parse_and_export_attraction(url: str):
attractions = []
for url in urls:
attractions.append(parse_location(url))
return attractions
def parse_location(url: str) -> TripadvisorAttraction:
parser = TripadvisorAttractionParser(url.strip()) parser = TripadvisorAttractionParser(url.strip())
return parser.parse() attraction = parser.parse()
export_attraction(attraction)
def get_urls() -> [str]: def get_urls() -> [str]:
@ -22,9 +16,8 @@ def get_urls() -> [str]:
def main(): def main():
attractions = parse_locations(get_urls()) with ThreadPoolExecutor() as executor:
for attraction in attractions: executor.map(parse_and_export_attraction, get_urls())
export_attraction(attraction)
if __name__ == "__main__": if __name__ == "__main__":