v0.5.1-beta

v0.5.1-beta

lib/collector/option/strategies/group.js

@@ -39,12 +39,12 @@ class GroupStrategy {
         let result = [];
 
         for(let option of element) {
-            
+
             const clickableElement = option.querySelector('input') || option;
 
             result.push({
                 value: clickableElement,
-                [identifier]: clickableElement
+                [identifier]: option
             });
         }
 

package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "@plato2/impressionist",
-  "version": "0.5.0-beta",
+  "version": "0.5.1-beta",
   "description": "Impressionist is a JavaScript library that allows you to scrape data in less time.",
   "main": "./src/process.js",
   "scripts": {
@@ -23,7 +23,7 @@
     "test-Query": "mocha --timeout 60000 ./test/collector/query/query.js",
     "test-querySelectors": "mocha --timeout 60000 ./test/collector/query/selectors/**",
     "test-queryInterpreter": "mocha --timeout 60000 --recursive ./test/collector/query/interpreter/**",
-    "test-scraper": "mocha --timeout 60000 ./test/scraper/main.js"
+    "test-scraper": "mocha --timeout 60000 ./test/scraper/*"
   },
   "repository": {
     "type": "git",

src/environment.js

@@ -27,7 +27,7 @@ class Environment {
     /**
      * The current version of the library. It can be used, for example, for bug reporting plugins.
      */
-    static IMPRESSIONIST_VERSION = '0.5.0-beta';
+    static IMPRESSIONIST_VERSION = '0.5.1-beta';
 
     /**
      * Check if a specific environment is running. i.e prod or dev.

test/scraper/main.js → test/scraper/scraper1.js

@@ -18,7 +18,7 @@ import assert from "assert";
 import Impressionist from '../../src/process.js';
 import NanoServer from '../testing-server/server.js';
 
-describe.only('Main Scrape Test', () => {
+describe('Scraping with local testing page and Select Options', () => {
 
     const testingServer = new NanoServer();
     const url = 'http://localhost:8081';
@@ -31,11 +31,6 @@ describe.only('Main Scrape Test', () => {
 
         const result = await Impressionist.execute(url, async (browser, page) => {
 
-            page.on('console', (msg) => {
-                for (let i = 0; i < msg.args().length; ++i)
-                  console.log(`${i}: ${msg.args()[i]}`);
-            });
-
             return await page.evaluate( async () => {
 
                 const data = collector({

test/scraper/scraper2.js

@@ -0,0 +1,69 @@
+/*
+ Copyright 2021 Plato Solutions, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+import assert from "assert";
+import Impressionist from '../../src/process.js';
+import NanoServer from '../testing-server/server.js';
+
+describe('Scraping with local testing page and Group Options', () => {
+
+    const testingServer = new NanoServer();
+    const url = 'http://localhost:8081';
+
+    before(async () => {
+        await testingServer.start();
+    });
+
+    it('Scrape Test Page', async () => {
+
+        const result = await Impressionist.execute(url, async (browser, page) => {
+
+            return await page.evaluate( async () => {
+
+                const data = collector({
+                    name: 'h1',
+                    reviews: elements('{#reviews > ul > li}*').iterate({
+                        title: '#review-title',
+                        author: '#review-author',
+                        rating: '#review-rating',
+                        date: '#review-date',
+                        body: '#review-body'
+                    }),
+                    bundles: options({
+                        edition: '{#div-1 > div}*', // #div-1 > div
+                        support: '{#div-2 > div}*',
+                        installation: '{#div-3 > div}*',
+                    }).iterate({
+                        price: '::document > #price-div'
+                    }).post(options => { 
+                        return options.map(({edition, support, installation, price}) => {
+                            return {edition, support, installation, price};
+                        });
+                    })
+                });
+
+                return await data.call();
+            });
+        });
+
+        assert.deepStrictEqual(result, {"name":"Plato Plugin","reviews":[{"title":"It is okay","author":"John Doe","rating":"4","date":"01-12-2021","body":"Nice product. I would recommend the version X."},{"title":"Amazing!","author":"Richard Roe","rating":"5","date":"10-12-2021","body":"Really good product."}],"bundles":[{"edition":"div-val-10","support":"div-val-40","installation":"div-val-60","price":"110"},{"edition":"div-val-10","support":"div-val-40","installation":"div-val-70","price":"120"},{"edition":"div-val-10","support":"div-val-40","installation":"div-val-80","price":"130"},{"edition":"div-val-10","support":"div-val-50","installation":"div-val-60","price":"120"},{"edition":"div-val-10","support":"div-val-50","installation":"div-val-70","price":"130"},{"edition":"div-val-10","support":"div-val-50","installation":"div-val-80","price":"140"},{"edition":"div-val-20","support":"div-val-40","installation":"div-val-60","price":"120"},{"edition":"div-val-20","support":"div-val-40","installation":"div-val-70","price":"130"},{"edition":"div-val-20","support":"div-val-40","installation":"div-val-80","price":"140"},{"edition":"div-val-20","support":"div-val-50","installation":"div-val-60","price":"130"},{"edition":"div-val-20","support":"div-val-50","installation":"div-val-70","price":"140"},{"edition":"div-val-20","support":"div-val-50","installation":"div-val-80","price":"150"},{"edition":"div-val-30","support":"div-val-40","installation":"div-val-60","price":"130"},{"edition":"div-val-30","support":"div-val-40","installation":"div-val-70","price":"140"},{"edition":"div-val-30","support":"div-val-40","installation":"div-val-80","price":"150"},{"edition":"div-val-30","support":"div-val-50","installation":"div-val-60","price":"140"},{"edition":"div-val-30","support":"div-val-50","installation":"div-val-70","price":"150"},{"edition":"div-val-30","support":"div-val-50","installation":"div-val-80","price":"160"}]});
+    });
+
+    after(async () => {
+        await testingServer.stop();
+    });
+
+});

test/scraper/scraper3.js

@@ -0,0 +1,69 @@
+/*
+ Copyright 2021 Plato Solutions, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+import assert from "assert";
+import Impressionist from '../../src/process.js';
+import NanoServer from '../testing-server/server.js';
+
+describe('Scraping with local testing page and mixed Options', () => {
+
+    const testingServer = new NanoServer();
+    const url = 'http://localhost:8081';
+
+    before(async () => {
+        await testingServer.start();
+    });
+
+    it('Scrape Test Page', async () => {
+
+        const result = await Impressionist.execute(url, async (browser, page) => {
+
+            return await page.evaluate( async () => {
+
+                const data = collector({
+                    name: 'h1',
+                    reviews: elements('{#reviews > ul > li}*').iterate({
+                        title: '#review-title',
+                        author: '#review-author',
+                        rating: '#review-rating',
+                        date: '#review-date',
+                        body: '#review-body'
+                    }),
+                    bundles: options({
+                        edition: '{#option-1}',
+                        support: '{#div-2 > div}*',
+                        installation: '{#toogle}',
+                    }).iterate({
+                        price: '::document > #price-div'
+                    }).post(options => { 
+                        return options.map(({edition, support, installation, price}) => {
+                            return {edition, support, installation, price};
+                        });
+                    })
+                });
+
+                return await data.call();
+            });
+        });
+
+        assert.deepStrictEqual(result, {"name":"Plato Plugin","reviews":[{"title":"It is okay","author":"John Doe","rating":"4","date":"01-12-2021","body":"Nice product. I would recommend the version X."},{"title":"Amazing!","author":"Richard Roe","rating":"5","date":"10-12-2021","body":"Really good product."}],"bundles":[{"edition":"val-10","support":"div-val-40","installation":true,"price":"110"},{"edition":"val-10","support":"div-val-40","price":"110"},{"edition":"val-10","support":"div-val-50","installation":true,"price":"120"},{"edition":"val-10","support":"div-val-50","price":"120"},{"edition":"val-20","support":"div-val-40","installation":true,"price":"110"},{"edition":"val-20","support":"div-val-40","price":"110"},{"edition":"val-20","support":"div-val-50","installation":true,"price":"120"},{"edition":"val-20","support":"div-val-50","price":"120"},{"edition":"val-30","support":"div-val-40","installation":true,"price":"110"},{"edition":"val-30","support":"div-val-40","price":"110"},{"edition":"val-30","support":"div-val-50","installation":true,"price":"120"},{"edition":"val-30","support":"div-val-50","price":"120"}]});
+    });
+
+    after(async () => {
+        await testingServer.stop();
+    });
+
+});