TUIS/Playwright_Scraper
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update scrape-within-domain-only.js

Browse Source 
fixed the crawler it should work as intended now
This commit is contained in:
Kиро.Kрика 2024-08-14 22:54:09 +03:00 committed by GitHub
parent 5dc63e2952
commit 8d48a9d811
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrape-within-domain-only.js
View File

@ -114,18 +114,25 @@ const ignoredDomainsRegex = /facebook\.com|linkedin\.com|youtube\.com|focus-news
// Find new links to crawl
const newLinks = [];
$('a').each((index, element) => {
const href = $(element).attr('href');
if (href && href.startsWith('http')) {
let href = $(element).attr('href');
if (href) {
href = new URL(href, currentPageUrl).href; // Convert to absolute URL
const linkUrlObj = new URL(href);
const linkHostname = linkUrlObj.hostname;
// Skip ignored domains and URLs outside the base domain
if (!ignoredDomainsRegex.test(linkHostname) && linkHostname === baseDomain) {
newLinks.push(href);
}
}
});
queue.push(...newLinks);
// Click on the "Next" button if present
try {
await page.click('selector-for-next-button');
} catch (clickError) {
console.log(`Next button not found on ${currentPageUrl} or error clicking it: ${clickError.message}`);
}
} catch (error) {
console.log(`Error loading or processing ${currentPageUrl}: ${error.message}`);
continue; // Skip to the next URL