1"""Module defines the main entry point for the Apify Actor.
2
3Feel free to modify this file to suit your specific needs.
4
5To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:
6https://docs.apify.com/sdk/python
7"""
8
9from __future__ import annotations
10
11import asyncio
12from urllib.parse import urljoin
13
14from apify import Actor, Request
15from selenium import webdriver
16from selenium.webdriver.chrome.options import Options as ChromeOptions
17from selenium.webdriver.common.by import By
18
19# To run this Actor locally, you need to have the Selenium Chromedriver installed.
20# Follow the installation guide at:
21# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/
22# When running on the Apify platform, the Chromedriver is already included
23# in the Actor's Docker image.
24
25
26asyncdefmain()->None:
27"""Define a main entry point for the Apify Actor.
28
29 This coroutine is executed using `asyncio.run()`, so it must remain an asynchronous function for proper execution.
30 Asynchronous execution is required for communication with Apify platform, and it also enhances performance in
31 the field of web scraping significantly.
32 """
33# Enter the context of the Actor.
34asyncwith Actor:
35# Retrieve the Actor input, and use default values if not provided.
36 actor_input =await Actor.get_input()or{}
37 start_urls = actor_input.get('start_urls',[{'url':'https://apify.com'}])
38 max_depth = actor_input.get('max_depth',1)
39
40# Exit if no start URLs are provided.
41ifnot start_urls:
42 Actor.log.info('No start URLs specified in Actor input, exiting...')
43await Actor.exit()
44
45# Open the default request queue for handling URLs to be processed.
46 request_queue =await Actor.open_request_queue()
47
48# Enqueue the start URLs with an initial crawl depth of 0.
49for start_url in start_urls:
50 url = start_url.get('url')
51 Actor.log.info(f'Enqueuing {url} ...')
52 new_request = Request.from_url(url, user_data={'depth':0})
53await request_queue.add_request(new_request)
54
55# Launch a new Selenium Chrome WebDriver and configure it.
56 Actor.log.info('Launching Chrome WebDriver...')
57 chrome_options = ChromeOptions()
58
59if Actor.configuration.headless:
60 chrome_options.add_argument('--headless')
61
62 chrome_options.add_argument('--no-sandbox')
63 chrome_options.add_argument('--disable-dev-shm-usage')
64 driver = webdriver.Chrome(options=chrome_options)
65
66# Test WebDriver setup by navigating to an example page.
67 driver.get('http://www.example.com')
68if driver.title !='Example Domain':
69raise ValueError('Failed to open example page.')
70
71# Process the URLs from the request queue.
72while request :=await request_queue.fetch_next_request():
73 url = request.url
74
75ifnotisinstance(request.user_data['depth'],(str,int)):
76raise TypeError('Request.depth is an enexpected type.')
77
78 depth =int(request.user_data['depth'])
79 Actor.log.info(f'Scraping {url} (depth={depth}) ...')
80
81try:
82# Navigate to the URL using Selenium WebDriver. Use asyncio.to_thread
83# for non-blocking execution.
84await asyncio.to_thread(driver.get, url)
85
86# If the current depth is less than max_depth, find nested links
87# and enqueue them.
88if depth < max_depth:
89for link in driver.find_elements(By.TAG_NAME,'a'):
90 link_href = link.get_attribute('href')
91 link_url = urljoin(url, link_href)
92
93if link_url.startswith(('http://','https://')):
94 Actor.log.info(f'Enqueuing {link_url} ...')
95 new_request = Request.from_url(
96 link_url,
97 user_data={'depth': depth +1},
98)
99await request_queue.add_request(new_request)
100
101# Extract the desired data.
102 data ={
103'url': url,
104'title': driver.title,
105}
106
107# Store the extracted data to the default dataset.
108await Actor.push_data(data)
109
110except Exception:
111 Actor.log.exception(f'Cannot extract data from {url}.')
112
113finally:
114# Mark the request as handled to ensure it is not processed again.
115await request_queue.mark_request_as_handled(request)
116
117 driver.quit()

A template example built with Selenium and a headless Chrome browser to scrape a website and save the results to storage. The URL of the web page is passed in via input, which is defined by the input schema . The template uses the Selenium WebDriver  to load and process the page. Enqueued URLs are stored in the default request queue . The data are then stored in the default dataset  where you can easily access them.

• Apify SDK  for Python - a toolkit for building Apify Actors  and scrapers in Python
• Input schema  - define and easily validate a schema for your Actor's input
• Request queue  - queues into which you can put the URLs you want to scrape
• Dataset  - store structured data where each object stored has the same attributes
• Selenium  - a browser automation library

This code is a Python script that uses Selenium to scrape web pages and extract data from them. Here's a brief overview of how it works:

• The script reads the input data from the Actor instance, which is expected to contain a start_urls key with a list of URLs to scrape and a max_depth key with the maximum depth of nested links to follow.
• The script enqueues the starting URLs in the default request queue and sets their depth to 1.
• The script processes the requests in the queue one by one, fetching the URL using requests and parsing it using Selenium.
• If the depth of the current request is less than the maximum depth, the script looks for nested links in the page and enqueues their targets in the request queue with an incremented depth.
• The script extracts the desired data from the page (in this case, titles of each page) and pushes them to the default dataset using the push_data method of the Actor instance.
• The script catches any exceptions that occur during the web scraping  process and logs an error message using the Actor.log.exception method.

• Selenium controlled Chrome example 
• Selenium Grid: what it is and how to set it up 
• Web scraping with Selenium and Python 
• Cypress vs. Selenium for web testing 
• Python tutorials in Academy 
• Video guide on getting scraped data using Apify API 
• A short guide on how to build web scrapers using code templates: