un_vr_scraper/scraper.py at master · egbertdev/un_vr_scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# scraper.py
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from database import insert_new_data
import time

def scrape_page_data(driver):
    """Scrape table data from the current page."""
    page_source = driver.page_source
    soup = BeautifulSoup(page_source, 'html.parser')

    # Find all rows in the table (modify this based on the actual table structure)
    rows = soup.find_all('tr')

    # Loop through each row and extract data
    for row in rows:
        # Extract data from each row (modify these selectors based on your table's structure)
        year = row.find('td', class_='views-field views-field-field-year')  # Adjust class name
        goverment = row.find('td', class_='views-field views-field-field-regional-local-government')  # Adjust class name
        country = row.find('td', class_='views-field views-field-field-country-value')  # Adjust class name
        report = row.find('td', class_='views-field views-field-field-fileurl')
        language = row.find('td', class_='views-field views-field-field-language')
        state = row.find('td', class_='views-field views-field-field-url')

        if year and goverment and country and report and language and state:  # Ensure that the row has all the columns
            year = year.get_text(strip=True)
            goverment = goverment.get_text(strip=True)
            country = country.get_text(strip=True)
            report = report.get_text(strip=True)
            language = language.get_text(strip=True)
            state = state.get_text(strip=True)
            # Insert data into database
            insert_new_data(year,goverment,country,report,language,state)
            #print(year,goverment,country,report,language,state)

def handle_pagination(driver):
    """Navigate through all pages and scrape data."""
    while True:
        # Wait for the table to load (adjust this based on the actual table)
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, 'table.cols-6'))
        )

        # Scrape data from the current page
        scrape_page_data(driver)

        # Check if there's a 'Next' button and click it, or break if no more pages
        try:
            next_button = next_button = driver.find_element(By.PARTIAL_LINK_TEXT, '››')
            driver.execute_script("arguments[0].scrollIntoView();", next_button)
            driver.execute_script("arguments[0].click();", next_button)
            time.sleep(5)  # Wait for the page to load before scraping again
        except:
            break