diff --git a/code/demo_automation.js b/code/demo_automation.js new file mode 100644 index 0000000..9f7efbd --- /dev/null +++ b/code/demo_automation.js @@ -0,0 +1,230 @@ +import puppeteer from "puppeteer"; + +const JOURNEY = { + DEPARTURE: "Cluj-Napoca", + ARRIVAL: "Bucuresti-Nord", + DEPARTURE_DATE: "20.04.2024", + DEPARTURE_TIME: 480, + ORDERING_TYPE: 2, +}; + +const USER_DETAILS = { + USERNAME: "", + PASSWORD: "", + CARD_NUMBER: "4242424242424242", + CARD_PERSON: "John Doe", + CARD_EXPIRATION_MONTH: "12", + CARD_EXPIRATION_YEAR: "2024", + CARD_CVV: "123", +}; + +const URL = `https://bilete.cfrcalatori.ro/ro-RO/Rute-trenuri/${JOURNEY.DEPARTURE}/${JOURNEY.ARRIVAL}?DepartureDate=${JOURNEY.DEPARTURE_DATE}&MinutesInDay=${JOURNEY.DEPARTURE_TIME}&OrderingTypeId=${JOURNEY.ORDERING_TYPE}`; + +const SELECTORS = { + TRAIN_PANEL: ".div-itinerary-station", + BUY_BUTTON: "#button-itinerary-0-buy", + TICKET_TYPE_NEXT_BUTTON: "#button-next-step-2", + TICKET_NUMBER_PLUS_BUTTON: "#button-ticket-fare-4-more", + TICKET_NUMBER_POPUP_BUTTON: "#div-warning-students-general button", + TICKET_NUMBER_NEXT_BUTTON: "#button-next-step-3", + PRICE_NEXT_BUTTON: "#button-next-step-4", + USERNAME_FIELD: "#UserName", + PASSWORD_FIELD: "#Password", + LOGIN_BUTTON: "#button-login", + YOUR_ACCOUNT_NEXT_BUTTON: "#button-next-step-5", + CONFIRM_BUTTON: "#button-confirm-selection", + CONFIRM_NEXT_BUTTON: "#button-next-step-6", + TRAVEL_DATA_PREFERENCES: "#button-load-preferences-0", + SELECT_PASSENGER_PREFERENCES: "#button-select-passenger-preference-0", + TRAVEL_DATA_NEXT_BUTTON: "#button-next-step-7", + SELECT_CARD_PAYMENT: "#ep-cc", + SELECT_PAY_ONLINE: ".btn-pay", + CARD_NUMBER: "#card", + CARD_NAME: "#name_on_card", + CARD_EXPIRING_MONTH: "#exp_month", + CARD_EXPIRING_YEAR: "#exp_year", + CARD_CVV: "#cvv2", + CARD_CONSENT: "#consent", + CARD_PAY_ONLINE: "#button_status", +}; + +function delay(time) { + return new Promise(function (resolve) { + setTimeout(resolve, time); + }); +} + +async function selectTrain(page) { + await page.waitForSelector(SELECTORS.TRAIN_PANEL); + + const train = await page.$(SELECTORS.TRAIN_PANEL); + + await train.waitForSelector(SELECTORS.BUY_BUTTON); + await delay(2000); + + const buyButton = await train.$(SELECTORS.BUY_BUTTON); + await buyButton.evaluate((button) => button.click()); +} + +async function selectTicketType(page) { + await page.waitForSelector(SELECTORS.TICKET_TYPE_NEXT_BUTTON); + + await delay(2000); + const nextButton = await page.$(SELECTORS.TICKET_TYPE_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function selectTicketNumber(page) { + await page.waitForSelector(SELECTORS.TICKET_NUMBER_PLUS_BUTTON); + + await delay(2000); + const plusButton = await page.$(SELECTORS.TICKET_NUMBER_PLUS_BUTTON); + await plusButton.evaluate((button) => button.click()); + + await page.waitForSelector(SELECTORS.TICKET_NUMBER_POPUP_BUTTON, { + visible: true, + }); + const popupButton = await page.$(SELECTORS.TICKET_NUMBER_POPUP_BUTTON); + await popupButton.evaluate((button) => button.click()); + + await page.waitForSelector(SELECTORS.TICKET_NUMBER_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.TICKET_NUMBER_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function confirmPrice(page) { + await page.waitForSelector(SELECTORS.PRICE_NEXT_BUTTON); + + await delay(2000); + const nextButton = await page.$(SELECTORS.PRICE_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function login(page) { + await page.waitForSelector(SELECTORS.USERNAME_FIELD); + await page.type(SELECTORS.USERNAME_FIELD, USER_DETAILS.USERNAME, { + delay: 100, + }); + + await page.waitForSelector(SELECTORS.PASSWORD_FIELD); + await page.type(SELECTORS.PASSWORD_FIELD, USER_DETAILS.PASSWORD, { + delay: 100, + }); + + await page.waitForSelector(SELECTORS.LOGIN_BUTTON); + const loginButton = await page.$(SELECTORS.LOGIN_BUTTON); + await loginButton.evaluate((button) => button.click()); + + await page.waitForSelector(SELECTORS.YOUR_ACCOUNT_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.YOUR_ACCOUNT_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function confirmBooking(page) { + try { + await page.waitForSelector(SELECTORS.CONFIRM_BUTTON, { + visible: true, + timeout: 5000, + }); + const confirmButton = await page.$(SELECTORS.CONFIRM_BUTTON); + await confirmButton.evaluate((el) => el.click()); + } catch (error) { + console.log("No confirm button"); + } + + console.log("Selection confirmed"); + await page.waitForFunction( + (selector) => { + const button = document.querySelector(selector); + return button && !button.disabled; + }, + { polling: "mutation" }, + SELECTORS.CONFIRM_NEXT_BUTTON, + ); + + await page.waitForSelector(SELECTORS.CONFIRM_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.CONFIRM_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function handlePayment(page) { + await page.waitForSelector(SELECTORS.SELECT_CARD_PAYMENT); + const selectCards = await page.$(SELECTORS.SELECT_CARD_PAYMENT); + await selectCards.evaluate((el) => el.click()); + + await page.waitForSelector(SELECTORS.SELECT_PAY_ONLINE); + const selectPayOnline = await page.$(SELECTORS.SELECT_PAY_ONLINE); + await selectPayOnline.evaluate((el) => el.click()); + + await page.waitForSelector(SELECTORS.CARD_NUMBER); + await page.type(SELECTORS.CARD_NUMBER, USER_DETAILS.CARD_NUMBER, { + delay: 100, + }); + await page.type(SELECTORS.CARD_NAME, USER_DETAILS.CARD_PERSON, { + delay: 100, + }); + await page.type( + SELECTORS.CARD_EXPIRING_MONTH, + USER_DETAILS.CARD_EXPIRATION_MONTH, + { delay: 100 }, + ); + await page.type( + SELECTORS.CARD_EXPIRING_YEAR, + USER_DETAILS.CARD_EXPIRATION_YEAR, + { delay: 100 }, + ); + await page.type(SELECTORS.CARD_CVV, USER_DETAILS.CARD_CVV, { delay: 100 }); + + await page.waitForSelector(SELECTORS.CARD_CONSENT); + const consent = await page.$(SELECTORS.CARD_CONSENT); + await consent.evaluate((el) => el.click()); + + await page.waitForSelector(SELECTORS.CARD_PAY_ONLINE); + const payOnline = await page.$(SELECTORS.CARD_PAY_ONLINE); + await payOnline.evaluate((el) => el.click()); +} + +async function selectStudentCard(page) { + await page.waitForSelector(SELECTORS.TRAVEL_DATA_PREFERENCES); + const travelDataPreferences = await page.$(SELECTORS.TRAVEL_DATA_PREFERENCES); + await travelDataPreferences.evaluate((button) => button.click()); + + await delay(2000); + + await page.waitForSelector(SELECTORS.SELECT_PASSENGER_PREFERENCES); + const selectPassengerPreferences = await page.$( + SELECTORS.SELECT_PASSENGER_PREFERENCES, + ); + await selectPassengerPreferences.evaluate((button) => button.click()); + + await delay(2000); + + await page.waitForSelector(SELECTORS.TRAVEL_DATA_NEXT_BUTTON); + const travelDataNextButton = await page.$(SELECTORS.TRAVEL_DATA_NEXT_BUTTON); + await travelDataNextButton.evaluate((button) => button.click()); +} + +async function run() { + const browser = await puppeteer.launch({ + headless: false, + defaultViewport: null, + args: ["--start-maximized"], + }); + const page = await browser.newPage(); + + await page.goto(URL); + + await selectTrain(page); + await selectTicketType(page); + await selectTicketNumber(page); + await confirmPrice(page); + await login(page); + await confirmBooking(page); + await selectStudentCard(page); + await handlePayment(page); +} + +run(); diff --git a/code/demo_ghost.js b/code/demo_ghost.js new file mode 100644 index 0000000..4baac56 --- /dev/null +++ b/code/demo_ghost.js @@ -0,0 +1,238 @@ +import puppeteer from "puppeteer"; +import ghostCursor from "ghost-cursor"; + +const JOURNEY = { + DEPARTURE: "Cluj-Napoca", + ARRIVAL: "Bucuresti-Nord", + DEPARTURE_DATE: "20.04.2024", + DEPARTURE_TIME: 480, + ORDERING_TYPE: 2, +}; + +const USER_DETAILS = { + USERNAME: "", + PASSWORD: "", + CARD_NUMBER: "4242424242424242", + CARD_PERSON: "John Doe", + CARD_EXPIRATION_MONTH: "12", + CARD_EXPIRATION_YEAR: "2024", + CARD_CVV: "123", +}; + +const URL = `https://bilete.cfrcalatori.ro/ro-RO/Rute-trenuri/${JOURNEY.DEPARTURE}/${JOURNEY.ARRIVAL}?DepartureDate=${JOURNEY.DEPARTURE_DATE}&MinutesInDay=${JOURNEY.DEPARTURE_TIME}&OrderingTypeId=${JOURNEY.ORDERING_TYPE}`; + +const SELECTORS = { + TRAIN_PANEL: ".div-itinerary-station", + BUY_BUTTON: "#button-itinerary-0-buy", + TICKET_TYPE_NEXT_BUTTON: "#button-next-step-2", + TICKET_NUMBER_PLUS_BUTTON: "#button-ticket-fare-4-more", + TICKET_NUMBER_POPUP_BUTTON: "#div-warning-students-general button", + TICKET_NUMBER_NEXT_BUTTON: "#button-next-step-3", + PRICE_NEXT_BUTTON: "#button-next-step-4", + USERNAME_FIELD: "#UserName", + PASSWORD_FIELD: "#Password", + LOGIN_BUTTON: "#button-login", + YOUR_ACCOUNT_NEXT_BUTTON: "#button-next-step-5", + CONFIRM_BUTTON: "#button-confirm-selection", + CONFIRM_NEXT_BUTTON: "#button-next-step-6", + TRAVEL_DATA_PREFERENCES: "#button-load-preferences-0", + SELECT_PASSENGER_PREFERENCES: "#button-select-passenger-preference-0", + TRAVEL_DATA_NEXT_BUTTON: "#button-next-step-7", + SELECT_CARD_PAYMENT: "#ep-cc", + SELECT_PAY_ONLINE: ".btn-pay", + CARD_NUMBER: "#card", + CARD_NAME: "#name_on_card", + CARD_EXPIRING_MONTH: "#exp_month", + CARD_EXPIRING_YEAR: "#exp_year", + CARD_CVV: "#cvv2", + CARD_CONSENT: "#consent", + CARD_PAY_ONLINE: "#button_status", +}; + +function delay(time) { + return new Promise(function (resolve) { + setTimeout(resolve, time); + }); +} + +async function selectTrain(page, cursor) { + await page.waitForSelector(SELECTORS.TRAIN_PANEL); + + const train = await page.$(SELECTORS.TRAIN_PANEL); + + await train.waitForSelector(SELECTORS.BUY_BUTTON); + await delay(2000); + + const buyButton = await train.$(SELECTORS.BUY_BUTTON); + await cursor.click(buyButton); +} + +async function selectTicketType(page, cursor) { + await page.waitForSelector(SELECTORS.TICKET_TYPE_NEXT_BUTTON); + + await delay(2000); + const nextButton = await page.$(SELECTORS.TICKET_TYPE_NEXT_BUTTON); + await cursor.click(nextButton); +} + +async function selectTicketNumber(page, cursor) { + await page.waitForSelector(SELECTORS.TICKET_NUMBER_PLUS_BUTTON); + + await delay(2000); + const plusButton = await page.$(SELECTORS.TICKET_NUMBER_PLUS_BUTTON); + await cursor.click(plusButton); + + await page.waitForSelector(SELECTORS.TICKET_NUMBER_POPUP_BUTTON, { + visible: true, + }); + await delay(2000); + const popupButton = await page.$(SELECTORS.TICKET_NUMBER_POPUP_BUTTON); + await cursor.click(popupButton); + + await page.waitForSelector(SELECTORS.TICKET_NUMBER_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.TICKET_NUMBER_NEXT_BUTTON); + await cursor.click(nextButton); +} + +async function confirmPrice(page, cursor) { + await page.waitForSelector(SELECTORS.PRICE_NEXT_BUTTON); + + await delay(2000); + const nextButton = await page.$(SELECTORS.PRICE_NEXT_BUTTON); + await cursor.click(nextButton); +} + +async function login(page, cursor) { + await page.waitForSelector(SELECTORS.USERNAME_FIELD); + await page.type(SELECTORS.USERNAME_FIELD, USER_DETAILS.USERNAME, { + delay: 100, + }); + + await page.waitForSelector(SELECTORS.PASSWORD_FIELD); + await page.type(SELECTORS.PASSWORD_FIELD, USER_DETAILS.PASSWORD, { + delay: 100, + }); + + await page.waitForSelector(SELECTORS.LOGIN_BUTTON); + const loginButton = await page.$(SELECTORS.LOGIN_BUTTON); + await cursor.click(loginButton); + + await page.waitForSelector(SELECTORS.YOUR_ACCOUNT_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.YOUR_ACCOUNT_NEXT_BUTTON); + await cursor.click(nextButton); +} + +async function confirmBooking(page, cursor) { + try { + await page.waitForSelector(SELECTORS.CONFIRM_BUTTON, { + visible: true, + timeout: 5000, + }); + const confirmButton = await page.$(SELECTORS.CONFIRM_BUTTON); + await cursor.click(confirmButton); + } catch (error) { + console.log("No confirm button"); + } + + console.log("Selection confirmed"); + await page.waitForFunction( + (selector) => { + const button = document.querySelector(selector); + return button && !button.disabled; + }, + { polling: "mutation" }, + SELECTORS.CONFIRM_NEXT_BUTTON, + ); + + await page.waitForSelector(SELECTORS.CONFIRM_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.CONFIRM_NEXT_BUTTON); + await cursor.click(nextButton); +} + +async function handlePayment(page, cursor) { + await page.waitForSelector(SELECTORS.SELECT_CARD_PAYMENT); + const selectCards = await page.$(SELECTORS.SELECT_CARD_PAYMENT); + await cursor.click(selectCards); + + await page.waitForSelector(SELECTORS.SELECT_PAY_ONLINE); + const selectPayOnline = await page.$(SELECTORS.SELECT_PAY_ONLINE); + await cursor.click(selectPayOnline); + + await page.waitForSelector(SELECTORS.CARD_NUMBER); + await page.type(SELECTORS.CARD_NUMBER, USER_DETAILS.CARD_NUMBER, { + delay: 100, + }); + await page.type(SELECTORS.CARD_NAME, USER_DETAILS.CARD_PERSON, { + delay: 100, + }); + await page.type( + SELECTORS.CARD_EXPIRING_MONTH, + USER_DETAILS.CARD_EXPIRATION_MONTH, + { delay: 100 }, + ); + await page.type( + SELECTORS.CARD_EXPIRING_YEAR, + USER_DETAILS.CARD_EXPIRATION_YEAR, + { delay: 100 }, + ); + await page.type(SELECTORS.CARD_CVV, USER_DETAILS.CARD_CVV, { delay: 100 }); + + await page.waitForSelector(SELECTORS.CARD_CONSENT); + const consent = await page.$(SELECTORS.CARD_CONSENT); + await cursor.click(consent); + + await page.waitForSelector(SELECTORS.CARD_PAY_ONLINE); + const payOnline = await page.$(SELECTORS.CARD_PAY_ONLINE); + await cursor.click(payOnline); +} + +async function selectStudentCard(page, cursor) { + await page.waitForSelector(SELECTORS.TRAVEL_DATA_PREFERENCES); + const travelDataPreferences = await page.$(SELECTORS.TRAVEL_DATA_PREFERENCES); + await cursor.click(travelDataPreferences); + + await delay(2000); + + await page.waitForSelector(SELECTORS.SELECT_PASSENGER_PREFERENCES); + const selectPassengerPreferences = await page.$( + SELECTORS.SELECT_PASSENGER_PREFERENCES, + ); + await cursor.click(selectPassengerPreferences); + + await delay(2000); + + await page.waitForSelector(SELECTORS.TRAVEL_DATA_NEXT_BUTTON); + const travelDataNextButton = await page.$(SELECTORS.TRAVEL_DATA_NEXT_BUTTON); + await cursor.click(travelDataNextButton); +} + +async function run() { + const browser = await puppeteer.launch({ + headless: false, + defaultViewport: null, + args: ["--start-maximized"], + }); + const page = await browser.newPage(); + const cursor = ghostCursor.createCursor( + page, + await ghostCursor.getRandomPagePoint(page), // start in a random position + ); + + await ghostCursor.installMouseHelper(page); + + await page.goto(URL); + + await selectTrain(page, cursor); + await selectTicketType(page, cursor); + await selectTicketNumber(page, cursor); + await confirmPrice(page, cursor); + await login(page, cursor); + await confirmBooking(page, cursor); + await selectStudentCard(page, cursor); + await handlePayment(page, cursor); +} + +run(); diff --git a/code/demo_scraping.js b/code/demo_scraping.js index a964006..753107f 100644 --- a/code/demo_scraping.js +++ b/code/demo_scraping.js @@ -33,16 +33,16 @@ async function run() { const title = await announcement.$eval( SELECTORS.TITLE, - (el) => el.textContent + (el) => el.textContent, ); const price = await announcement.$eval( SELECTORS.PRICE, - (el) => el.textContent + (el) => el.textContent, ); try { surface = await announcement.$eval( SELECTORS.SURFACE, - (el) => el.textContent + (el) => el.textContent, ); } catch (e) { // do nothing @@ -57,7 +57,7 @@ async function run() { try { const nextPageURL = await page.$eval( SELECTORS.NEXT_PAGE, - (el) => el.href + (el) => el.href, ); await page.goto(nextPageURL); } catch (e) { diff --git a/code/exercise_scraping.js b/code/exercise_scraping.js index 5f7101d..7944fe3 100644 --- a/code/exercise_scraping.js +++ b/code/exercise_scraping.js @@ -41,18 +41,18 @@ async function run() { const location = await event.$eval( SELECTORS.LOCATION, - (el) => el.innerText + (el) => el.innerText, ); const description = await event.$eval( SELECTORS.DESCRIPTION, - (el) => el.innerText + (el) => el.innerText, ); const image = await event.$eval(SELECTORS.IMAGE, (el) => el.src); const tags = await event.$$eval(SELECTORS.TAGS, (tags) => - tags.map((tag) => tag.innerText) + tags.map((tag) => tag.innerText), ); data.push({ diff --git a/src/README.md b/src/README.md index 77dd411..2dacf24 100644 --- a/src/README.md +++ b/src/README.md @@ -7,3 +7,5 @@ Just as Git revolutionizes collaboration in the coding world, Puppeteer empowers Throughout this workshop, we'll dive into the intricacies of Puppeteer, unraveling its capabilities, exploring practical use cases, and equipping you with the skills to wield this tool effectively. Whether you're a seasoned developer or a curious novice, Puppeteer offers a gateway to a world where automation is not just a convenience but a game-changer. As promised, we'll look into ways of finding rent in Cluj Napoca in a more efficient way. So, get ready to dive into the world of Puppeteer, where lines of code become puppet strings, and the web transforms into your stage. Let's embark on this journey together, where every click, every scroll, and every interaction is powered by the magic of Puppeteer. Your apartment in Cluj Napoca awaits! + +### All the solutions are available here: [Puppeteer Workshop Solutions](https://github.com/UBBGDSC/puppeteer-workshop) diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 7c5a6b0..4c6e364 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -4,8 +4,8 @@ # What is Puppeteer? -- [Puppeteer introduction](./chapter-1.md) -- [Getting started](./chapter-1.1.md) +- [Puppeteer introduction](./what-is-puppeteer.md) +- [Getting started](./getting-started.md) - [Core concepts](./core-concepts/core.md) - [Browser Management](./core-concepts/browser-management.md) - [Page Interactions](./core-concepts/page-interactions.md) @@ -16,3 +16,6 @@ - [Exercise: Web Scraping](./exercise-scraping.md) --- + +- [Demo: Web Automation](./web-automation.md) +- [Confirm you are not a robot](./human-behaviour.md) diff --git a/src/chapter-2.1.md b/src/chapter-2.1.md deleted file mode 100644 index 10dc9d9..0000000 --- a/src/chapter-2.1.md +++ /dev/null @@ -1,66 +0,0 @@ -# Collaboration in Git - -## Merging - -Having multiple branches is extremely convenient to keep new changes separated and avoid pushing unapproved or broken changes to production. When the changes are approved, integrating them into the production branch is essential. - -### Fast-forward (--ff) - -A fast-forward merge occurs when the current branch has no additional commits compared to the branch being merged. Git prefers this as the easiest option: it merges the commit(s) from the branch being merged into the current branch without creating a new commit. - -![Fast-forward Merge](https://res.cloudinary.com/practicaldev/image/fetch/s--cT4TSe48--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/894znjv4oo9agqiz4dql.gif) - -### No-fast-forward (--no-ff) - -In case the current branch has commits not present in the branch to be merged, Git performs a no-fast-forward merge. This creates a new merging commit on the active branch, integrating changes from both branches. - -![No-fast-forward Merge](https://res.cloudinary.com/practicaldev/image/fetch/s--zRZ0x2Vc--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/rf1o2b6eduboqwkigg3w.gif) - -### Merge Conflicts - -Merging branches with conflicting changes in the same lines or with deleted files that were modified elsewhere can cause merge conflicts. Git highlights these conflicts, enabling manual resolution by removing unwanted changes, saving, re-adding the changed file, and committing the changes. - -![Merge Conflicts](https://res.cloudinary.com/practicaldev/image/fetch/s--7lBksXwA--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/bcd5ajtoc0g5dxzmpfbq.gif) - -### Rebasing - -Git rebase copies commits from the current branch and places them on top of the specified branch. Unlike merging, rebasing doesn't attempt to decide which files to keep, maintaining the branch's latest changes and avoiding merging conflicts. - -![Rebasing](https://res.cloudinary.com/practicaldev/image/fetch/s--EIY4OOcE--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/dwyukhq8yj2xliq4i50e.gif) - -Rebasing is ideal when integrating updates from the master branch into a feature branch, preventing future merging conflicts. - -### Interactive Rebase - -Interactive rebase allows modification of commits before rebasing, offering six actions: - -- reword: Edit the commit message. -- edit: Amend the commit. -- squash: Combine the commit into the previous one. -- fixup: Merge the commit into the previous one without retaining the commit's log message. -- exec: Execute a command on each commit being rebased. -- drop: Remove the commit. - -![Interactive Rebase](https://res.cloudinary.com/practicaldev/image/fetch/s--P6jr7igd--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/msofpv7k6rcmpaaefscm.gif) - -Feel free to use these actions for full control over your commits, whether removing or merging them. Interactive rebase facilitates the modification of commits. - -### Soft Reset - -A soft reset shifts HEAD to the specified commit without discarding changes introduced by the following commits. It preserves access to the changes made in the previous commits, enabling fixing and recommitting them later. - -![Soft Reset](https://res.cloudinary.com/practicaldev/image/fetch/s---GveiZe---/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/je5240aqa5uw9d8j3ibb.gif) - -### Hard Reset - -A hard reset returns Git's state to the specified commit, including changes in the working directory and staged files, discarding all changes introduced after the commit. - -![Hard Reset](https://res.cloudinary.com/practicaldev/image/fetch/s--GqjwnYkF--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/hlh0kowt3hov1xhcku38.gif) - -### Reverting - -Git revert creates a new commit that undoes the changes introduced by the specified commit, ensuring the commit history remains unaltered. - -![Reverting](https://res.cloudinary.com/practicaldev/image/fetch/s--eckmvr2M--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_66%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/i/3kkd2ahn41zixs12xgpf.gif) - -Git revert is useful for undoing a specific commit without altering the branch's history. diff --git a/src/exercise-2/add-personal-info.md b/src/exercise-2/add-personal-info.md deleted file mode 100644 index ff44e20..0000000 --- a/src/exercise-2/add-personal-info.md +++ /dev/null @@ -1 +0,0 @@ -# Add Personal Information diff --git a/src/exercise-2/clone-repo.md b/src/exercise-2/clone-repo.md deleted file mode 100644 index 5894c81..0000000 --- a/src/exercise-2/clone-repo.md +++ /dev/null @@ -1 +0,0 @@ -# Clone Repository diff --git a/src/exercise-2/create-pull-requests.md b/src/exercise-2/create-pull-requests.md deleted file mode 100644 index dcb70cb..0000000 --- a/src/exercise-2/create-pull-requests.md +++ /dev/null @@ -1 +0,0 @@ -# Create Pull Requests diff --git a/src/exercise-2/exercise-2.md b/src/exercise-2/exercise-2.md deleted file mode 100644 index 23d5d00..0000000 --- a/src/exercise-2/exercise-2.md +++ /dev/null @@ -1 +0,0 @@ -# Exercise: Merge Conflicts diff --git a/src/exercise-2/interactive-rebase.md b/src/exercise-2/interactive-rebase.md deleted file mode 100644 index e9d51f9..0000000 --- a/src/exercise-2/interactive-rebase.md +++ /dev/null @@ -1 +0,0 @@ -# Commit History and Interactive Rebase diff --git a/src/exercise-2/rebase-squash-merge.md b/src/exercise-2/rebase-squash-merge.md deleted file mode 100644 index 11267cf..0000000 --- a/src/exercise-2/rebase-squash-merge.md +++ /dev/null @@ -1 +0,0 @@ -# Rebase, Squash, and Merge diff --git a/src/exercise-2/resolve-merge-conflicts.md b/src/exercise-2/resolve-merge-conflicts.md deleted file mode 100644 index 1605c0c..0000000 --- a/src/exercise-2/resolve-merge-conflicts.md +++ /dev/null @@ -1 +0,0 @@ -# Resolve Merge Conflicts diff --git a/src/chapter-1.1.md b/src/getting-started.md similarity index 100% rename from src/chapter-1.1.md rename to src/getting-started.md diff --git a/src/human-behaviour.md b/src/human-behaviour.md new file mode 100644 index 0000000..7cf04ea --- /dev/null +++ b/src/human-behaviour.md @@ -0,0 +1,71 @@ +# Confirm you are not a robot + +Automations are great, but sometimes they can be too good, and be used for not-so-great purposes. Most of the browsers have mechanisms to detect if the user is a human or a robot. There are some libraries that can help you to bypass these mechanisms, but they are not recommended to be used in production environments. + +## [Ghost Cursor](https://github.com/Xetera/ghost-cursor) + +Remember the click action we saw in the previous chapter? While it's a great way to interact with the page, it's not the most human-like way to do it. The cursor literally teleports to the position and clicks. This is not how a human would interact with the page. Ghost Cursor is a library that simulates human-like cursor movements. It's a great way to make your automations more human-like. + +## How does it work + +Bezier curves do almost all the work here. They let us create an infinite amount of curves between any 2 points we want +and they look quite human-like. + +![](https://mamamoo.xetera.dev/😽🤵👲🧦👵.png) + +The magic comes from being able to set multiple points for the curve to go through. This is done by picking +2 coordinates randomly in a limited area above and under the curve. + + + +However, we don't want wonky looking cubic curves when using this method because nobody really moves their mouse +that way, so only one side of the line is picked when generating random points. + + +When calculating how fast the mouse should be moving we use Fitts's Law +to determine the amount of points we should be returning relative to the width of the element being clicked on and the distance +between the mouse and the object. + +## Setting up + +To install Ghost Cursor, run the following command: + +```bash +npm install ghost-cursor +``` + +## Usage + +We'll set up ghost cursor for our previous example. We'll use the same code as before, but we'll replace the `page.click` with `cursor.click`. + +By default, the cursor is not visible. For making it visible, we can use the function `cursor.installMouseHelper()`. This is super useful for debugging purposes. + +```javascript +async function run() { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto("https://example.com"); + const cursor = ghostCursor.createCursor( + page, + await ghostCursor.getRandomPagePoint(page), // start in a random position + true, // do random movements while moving + ); + await ghostCursor.installMouseHelper(page); +} +``` + +To use it, we'll need to send it as a parameter to the functions, together with the page object. For example, the function for selecting the train station would look like this: + +```javascript +async function selectTrain(page, cursor) { + await page.waitForSelector(SELECTORS.TRAIN_PANEL); + + const train = await page.$(SELECTORS.TRAIN_PANEL); + + await train.waitForSelector(SELECTORS.BUY_BUTTON); + await delay(2000); + + const buyButton = await train.$(SELECTORS.BUY_BUTTON); + await cursor.click(buyButton); +} +``` diff --git a/src/images/card_details.png b/src/images/card_details.png new file mode 100644 index 0000000..e701433 Binary files /dev/null and b/src/images/card_details.png differ diff --git a/src/images/card_payment_method.png b/src/images/card_payment_method.png new file mode 100644 index 0000000..a006d98 Binary files /dev/null and b/src/images/card_payment_method.png differ diff --git a/src/images/cfr_main_search.png b/src/images/cfr_main_search.png new file mode 100644 index 0000000..8018499 Binary files /dev/null and b/src/images/cfr_main_search.png differ diff --git a/src/images/confirmation.png b/src/images/confirmation.png new file mode 100644 index 0000000..561fd36 Binary files /dev/null and b/src/images/confirmation.png differ diff --git a/src/images/login.png b/src/images/login.png new file mode 100644 index 0000000..80581a2 Binary files /dev/null and b/src/images/login.png differ diff --git a/src/images/preferences.png b/src/images/preferences.png new file mode 100644 index 0000000..4a3085e Binary files /dev/null and b/src/images/preferences.png differ diff --git a/src/images/select_number_of_tickets.png b/src/images/select_number_of_tickets.png new file mode 100644 index 0000000..b15cb31 Binary files /dev/null and b/src/images/select_number_of_tickets.png differ diff --git a/src/images/select_type.png b/src/images/select_type.png new file mode 100644 index 0000000..523c33c Binary files /dev/null and b/src/images/select_type.png differ diff --git a/src/quiz.md b/src/quiz.md deleted file mode 100644 index cc9c719..0000000 --- a/src/quiz.md +++ /dev/null @@ -1 +0,0 @@ -# Quiz & Feedback diff --git a/src/web-automation.md b/src/web-automation.md new file mode 100644 index 0000000..9c2bf1a --- /dev/null +++ b/src/web-automation.md @@ -0,0 +1,432 @@ +# Demo: Web Automation + +Automations are a great way to save time and effort for repetitive tasks. In this demo, we will automate the process of purchasing a train ticket using Puppeteer. We will navigate to the [CFR website](https://www.irctc.co.in/nget/train-search), search for a train, and book a ticket. + +For this demo, we will be using the advanced search feature available on the website. It allows us to sort the data based on various parameters like the duration of the journey, the departure time, and the arrival time. + +CFR Main Search + +## Criteria + +We will search for a train from Cluj-Napoca to Bucharest. We will filter the results based on the following criteria: + +- The journey duration should be as short as possible. +- The departure time should be after 8:00 AM. + +## Journey Details + +To make things easier, we'll complete the form in the website. The generated link is + +https://bilete.cfrcalatori.ro/ro-RO/Rute-trenuri/Cluj-Napoca/Bucuresti-Nord?DepartureDate=20.04.2024&MinutesInDay=480&OrderingTypeId=2 + +We can observe that: + +- In the url, we have the departure and arrival stations +- We have a parameter for the departure date, represented in the format dd.mm.yyyy +- We have a parameter for the time, represented in minutes since midnight. For example, 480 minutes is 8:00 AM, 720 minutes is 12:00 PM, and so on. +- We have a parameter for the ordering type, which in our case is 2, meaning that we want to sort the results by the duration of the journey. + +We can extract those values as parameters in our script and generate the link dynamically, thus allowing us to buy tickets for multiple dates, times, and stations. + +```javascript +import puppeteer from "puppeteer"; + +const JOURNEY = { + DEPARTURE: "Cluj-Napoca", + ARRIVAL: "Bucuresti-Nord", + DEPARTURE_DATE: "20.04.2024", + DEPARTURE_TIME: 480, + ORDERING_TYPE: 2, +}; + +const URL = `https://bilete.cfrcalatori.ro/ro-RO/Rute-trenuri/${JOURNEY.DEPARTURE}/${JOURNEY.ARRIVAL}?DepartureDate=${JOURNEY.DEPARTURE_DATE}&MinutesInDay=${JOURNEY.DEPARTURE_TIME}&OrderingTypeId=${JOURNEY.ORDERING_TYPE}`; +``` + +In the code snippet above, we have defined the journey details and the URL for the search. We will use these constants in the subsequent chapters to automate the process of purchasing a train ticket. These can be easily modified to search for different journeys based on an user input. + +## Selectors + +Like in the previous chapters, we will define the selectors for the different elements on the page. We will use these selectors to interact with the page and extract the necessary information. + +At a first glance, we see that we need selectors to interact with the following elements: + +- Panels for the train +- Button to book the ticket + +There will be more selectors updated later in the chapter. + +```javascript +const SELECTORS = { + TRAIN_PANEL: ".train-panel", + BOOK_BUTTON: ".book-button", +}; +``` + +## Delay function + +In an automation, we'll interact directly with the elements, so we need to ensure that they are completely loaded before we interact with them. We can use a delay function to wait for a specified amount of time before proceeding with the next action. + +```javascript +function delay(time) { + return new Promise(function (resolve) { + setTimeout(resolve, time); + }); +} +``` + +In the code snippet above, we have defined a `delay` function that returns a promise which resolves after the specified amount of time. We will use this function to wait for the elements to load before any interaction. + +## Launching the browser + +For better visualization, we will use the `headless: false` option to see the browser in action. + +```javascript +async function run() { + const browser = await puppeteer.launch({ + headless: false, + defaultViewport: null, + args: ["--start-maximized"], + }); + const page = await browser.newPage(); + + await page.goto(URL); +} +``` + +## Selecting the train + +In this exercise, we will select the first train from the list, since it is the one with the shortest journey duration. We will click on the "Cumpără" button to proceed with the booking. + +#### Disclaimer: There should be some extra checks, to ensure that the train is not too early or too late, but for the sake of simplicity, we will assume that the first train is the best one. + +We will write a new function + +```javascript +async function selectTrain(page) { + await page.waitForSelector(SELECTORS.TRAIN_PANEL); + + const train = await page.$(SELECTORS.TRAIN_PANEL); + + await train.waitForSelector(SELECTORS.BUY_BUTTON); + await delay(2000); + + const buyButton = await train.$(SELECTORS.BUY_BUTTON); + await buyButton.evaluate((button) => button.click()); +} + +async funcion run() +{ + // previous code + await selectTrain(page); +} +``` + +## Ticket type + +Now, we are redirected to another page, where we can select the type of ticket we want to buy. We're happy with the default selection, so we will click on "Pasul următor", to proceed with the booking. + +CFR Ticket Type + +```javascript +const SELECTORS = { + // previous selectors + TICKET_TYPE_NEXT_BUTTON: ".next-button", +}; + +async function selectTicketType(page) { + await page.waitForSelector(SELECTORS.TICKET_TYPE_NEXT_BUTTON); + + await delay(2000); + const nextButton = await page.$(SELECTORS.TICKET_TYPE_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function run() { + // previous code + await selectTicketType(page); +} +``` + +## Number of tickets + +On the third step, we need to select the number of tickets we want to purchase. We have 2 options: Either we write the value corresponding to the number of tickets we want to buy, or we click on the "+" button to increase the number of tickets. We will choose the second option. + +CFR Ticket Total + +We can see that, after we click on the "+" button, a modal will appear, where we need to confirm something. We'll click on the "Am înțeles" button to proceed. + +```javascript +const SELECTORS = { + // previous selectors + TICKET_NUMBER_PLUS_BUTTON: ".plus-button", + TICKET_NUMBER_POPUP_BUTTON: ".popup-button", + TICKET_NUMBER_NEXT_BUTTON: ".next-button", +}; + +async function selectTicketNumber(page) { + await page.waitForSelector(SELECTORS.TICKET_NUMBER_PLUS_BUTTON); + + await delay(2000); + const plusButton = await page.$(SELECTORS.TICKET_NUMBER_PLUS_BUTTON); + await plusButton.evaluate((button) => button.click()); + + await page.waitForSelector(SELECTORS.TICKET_NUMBER_POPUP_BUTTON, { + visible: true, + }); + const popupButton = await page.$(SELECTORS.TICKET_NUMBER_POPUP_BUTTON); + await popupButton.evaluate((button) => button.click()); + + await page.waitForSelector(SELECTORS.TICKET_NUMBER_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.TICKET_NUMBER_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function run() { + // previous code + await selectTicketNumber(page); +} +``` + +## Price + +Probably the easiest step, we just need to click on the "Pasul următor" button to proceed. + +```javascript +const SELECTORS = { + // previous selectors + PRICE_NEXT_BUTTON: ".next-button", +}; + +async function selectPrice(page) { + await page.waitForSelector(SELECTORS.PRICE_NEXT_BUTTON); + + await delay(2000); + const nextButton = await page.$(SELECTORS.PRICE_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function run() { + // previous code + await selectPrice(page); +} +``` + +## Login + +Before purchasing the ticket, we need to login. Those will be set up as constants in the script. For this step, we need to target the username and password fields, and the login button. We will use the `type` function to fill in the fields. After we login, we will click on the "Pasul următor" button to proceed. + +Login + +```javascript +const USER_DETAILS = { + USERNAME: "your_username", + PASSWORD: "your_password", +}; + +const SELECTORS = { + // previous selectors + USERNAME_FIELD: "#usernameId", + PASSWORD_FIELD: "#passwordId", + LOGIN_BUTTON: ".login-button", + YOUR_ACCOUNT_NEXT_BUTTON: ".next-button", +}; + +async function login(page) { + await page.waitForSelector(SELECTORS.USERNAME_FIELD); + await page.type(SELECTORS.USERNAME_FIELD, USER_DETAILS.USERNAME, { + delay: 100, + }); + + await page.waitForSelector(SELECTORS.PASSWORD_FIELD); + await page.type(SELECTORS.PASSWORD_FIELD, USER_DETAILS.PASSWORD, { + delay: 100, + }); + + await page.waitForSelector(SELECTORS.LOGIN_BUTTON); + const loginButton = await page.$(SELECTORS.LOGIN_BUTTON); + await loginButton.evaluate((button) => button.click()); + + await page.waitForSelector(SELECTORS.YOUR_ACCOUNT_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.YOUR_ACCOUNT_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function run() { + // previous code + await login(page); +} +``` + +## Confirm Selection + +On this step, we need to confirm our selection. After clicking, we will click on the "Pasul următor" button to proceed. Until we confirm, we will not be able to proceed. For this, we'll use a function we did not mention before, the `waitForFunction` function. + +Confirm + +```javascript +const SELECTORS = { + // previous selectors + CONFIRM_SELECTION_BUTTON: ".confirm-button", + CONFIRM_SELECTION_NEXT_BUTTON: ".next-button", +}; + +async function confirmBooking(page) { + try { + await page.waitForSelector(SELECTORS.CONFIRM_BUTTON, { + visible: true, + timeout: 5000, + }); + const confirmButton = await page.$(SELECTORS.CONFIRM_BUTTON); + await confirmButton.evaluate((el) => el.click()); + } catch (error) { + console.log("No confirm button"); + } + + console.log("Selection confirmed"); + await page.waitForFunction( + (selector) => { + const button = document.querySelector(selector); + return button && !button.disabled; + }, + { polling: "mutation" }, + SELECTORS.CONFIRM_NEXT_BUTTON, + ); + + await page.waitForSelector(SELECTORS.CONFIRM_NEXT_BUTTON); + await delay(2000); + const nextButton = await page.$(SELECTORS.CONFIRM_NEXT_BUTTON); + await nextButton.evaluate((button) => button.click()); +} + +async function run() { + // previous code + await confirmBooking(page); +} +``` + +What does `waitForFunction` do? It waits for a function to return a truthy value. In our case, we are waiting for the confirm button to be enabled. We are using the `polling` option to check the condition every time the DOM is mutated. This is useful when we are waiting for an element to change its state. + +## Travel Data + +Thanks to the generosity of the Romanian Government, we have 90% discount for students. For this, we need to have the student card details linked to the account. In this sections, we can select from our preferences the student card, being the last step required to purchase the ticket. + +When we click to select the student card, a modal will appear, where we need to confirm our selection. There can be multiple student cards, so we need to select the first one. After we confirm, we will click on the "Spre plată" button to proceed. + +Login + +```javascript +const SELECTORS = { + // previous selectors + TRAVEL_DATA_PREFERENCES: "" + SELECT_PASSENGER_PREFERENCES: "", + TRAVEL_DATA_NEXT_BUTTON: "", +}; + +async function selectStudentCard(page) { + await page.waitForSelector(SELECTORS.TRAVEL_DATA_PREFERENCES); + const travelDataPreferences = await page.$(SELECTORS.TRAVEL_DATA_PREFERENCES); + await travelDataPreferences.evaluate((button) => button.click()); + + await delay(2000); + + await page.waitForSelector(SELECTORS.SELECT_PASSENGER_PREFERENCES); + const selectPassengerPreferences = await page.$( + SELECTORS.SELECT_PASSENGER_PREFERENCES + ); + await selectPassengerPreferences.evaluate((button) => button.click()); + + await delay(2000); + + await page.waitForSelector(SELECTORS.TRAVEL_DATA_NEXT_BUTTON); + const travelDataNextButton = await page.$(SELECTORS.TRAVEL_DATA_NEXT_BUTTON); + await travelDataNextButton.evaluate((button) => button.click()); +} + +async function run() { + // previous code + await selectStudentCard(page); +} +``` + +We're finally at the payment step. For this, we'll use fake data, since we're not actually going to buy the ticket. + +## Payment + +The payment is composed of 2 parts: + +- Selecting to pay using an online credit card +- Filling the form with the card details + +Login +Login + +```javascript +const USER_DETAILS = { + // previous user details + CARD_NUMBER: "1234 5678 1234 5678", + CARD_PERSON: "John Doe", + CARD_EXPIRATION_MONTH: "12", + CARD_EXPIRATION_YEAR: "2024", + CARD_CVV: "123", +}; + +const SELECTORS = { + // previous selectors + SELECT_CARD_PAYMENT: "", + SELECT_PAY_ONLINE: "", + CARD_NUMBER: "", + CARD_NAME: "", + CARD_EXPIRING_MONTH: "", + CARD_EXPIRING_YEAR: "", + CARD_CVV: "", + CARD_CONSENT: "", + CARD_PAY_ONLINE: "", +}; + +async function handlePayment(page) { + await page.waitForSelector(SELECTORS.SELECT_CARD_PAYMENT); + const selectCards = await page.$(SELECTORS.SELECT_CARD_PAYMENT); + await selectCards.evaluate((el) => el.click()); + + await page.waitForSelector(SELECTORS.SELECT_PAY_ONLINE); + const selectPayOnline = await page.$(SELECTORS.SELECT_PAY_ONLINE); + await selectPayOnline.evaluate((el) => el.click()); + + await page.waitForSelector(SELECTORS.CARD_NUMBER); + await page.type(SELECTORS.CARD_NUMBER, USER_DETAILS.CARD_NUMBER, { + delay: 100, + }); + await page.type(SELECTORS.CARD_NAME, USER_DETAILS.CARD_PERSON, { + delay: 100, + }); + await page.type( + SELECTORS.CARD_EXPIRING_MONTH, + USER_DETAILS.CARD_EXPIRATION_MONTH, + { delay: 100 }, + ); + await page.type( + SELECTORS.CARD_EXPIRING_YEAR, + USER_DETAILS.CARD_EXPIRATION_YEAR, + { delay: 100 }, + ); + await page.type(SELECTORS.CARD_CVV, USER_DETAILS.CARD_CVV, { delay: 100 }); + + await page.waitForSelector(SELECTORS.CARD_CONSENT); + const consent = await page.$(SELECTORS.CARD_CONSENT); + await consent.evaluate((el) => el.click()); + + await page.waitForSelector(SELECTORS.CARD_PAY_ONLINE); + const payOnline = await page.$(SELECTORS.CARD_PAY_ONLINE); + await payOnline.evaluate((el) => el.click()); +} + +async function run() { + // previous code + await handlePayment(page); + await browser.close(); +} +``` + +Congratulations! You have successfully automated the process of purchasing a train ticket using Puppeteer. You can now run the script and see the magic happen. diff --git a/src/web-scraping.md b/src/web-scraping.md index dc9c707..84d32bf 100644 --- a/src/web-scraping.md +++ b/src/web-scraping.md @@ -63,16 +63,16 @@ for (const announcement of announcements) { const title = await announcement.$eval( SELECTORS.TITLE, - (el) => el.textContent + (el) => el.textContent, ); const price = await announcement.$eval( SELECTORS.PRICE, - (el) => el.textContent + (el) => el.textContent, ); try { surface = await announcement.$eval( SELECTORS.SURFACE, - (el) => el.textContent + (el) => el.textContent, ); } catch (e) { // do nothing diff --git a/src/chapter-1.md b/src/what-is-puppeteer.md similarity index 100% rename from src/chapter-1.md rename to src/what-is-puppeteer.md