Alter 3 files
Update `.gitignore` Update `__main__.py` Update `__main__.py`
This commit is contained in:
parent
35735466c0
commit
129a2f5605
3 changed files with 23 additions and 10 deletions
|
@ -1,29 +1,36 @@
|
|||
import scrapers
|
||||
from selenium import webdriver
|
||||
from selenium.common.exceptions import WebDriverException
|
||||
from tqdm import tqdm
|
||||
import json
|
||||
import time
|
||||
import sys
|
||||
|
||||
OUTPUTFILE = sys.argv[0]
|
||||
OUTPUTFILE = sys.argv[1]
|
||||
|
||||
driver = webdriver.Firefox()
|
||||
driver_options = webdriver.ChromeOptions()
|
||||
driver_options.add_argument("--headless=new")
|
||||
driver_options.add_argument("--window-size=1920,1080")
|
||||
driver = webdriver.Chrome(options=driver_options)
|
||||
|
||||
datapoints = []
|
||||
|
||||
try:
|
||||
for cls in tqdm(
|
||||
for i, cls in enumerate(tqdm(
|
||||
[
|
||||
scrapers.Minilab,
|
||||
scrapers.TheFilmSafe,
|
||||
scrapers.HarmanLab,
|
||||
scrapers.AGPhotoLab,
|
||||
scrapers.FilmProcessingCoUk,
|
||||
scrapers.PPPCamera,
|
||||
scrapers.AnalogueWonderland,
|
||||
scrapers.Minilab,
|
||||
]
|
||||
):
|
||||
)):
|
||||
datapoints += cls(driver).scrape()
|
||||
except WebDriverException as e:
|
||||
driver.save_screenshot(f"crash.{int(time.time())}.png")
|
||||
raise e
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue