diff --git a/README.md b/README.md index 9eacabf..1c66f94 100755 --- a/README.md +++ b/README.md @@ -8,17 +8,18 @@ The goal of the test suite project is to produce test suites that can be used to verify the compliance of any server implementation to the Solid specifications. -To accomplish this, the project will in part reuse existing test -suites that have been developed for technologies Solid depends on, for -example the W3C LDP testsuite. - -In part, the project will develop test suites specifically for Solid -technologies. Since most Solid specifications do not require extensive -program flow to be tested, the main line of work is to develop a -system that uses RDF to describe the tests, so that most of the -tests are developed using RDF. This does not exclude the possibility -of writing a test framework using a conventional programming language -for parts of Solid that would be better served by it. +For the most part, the project will develop test suites specifically +for Solid technologies. Since most Solid specifications do not require +extensive program flow to be tested, the main line of work is to +develop a system that uses RDF to describe the tests, so that most of +the tests are developed using RDF. This does not exclude the +possibility of writing a test framework using a conventional +programming language for parts of Solid that would be better served by +it. + +Secondly, the project may integrate test suites from projects that +Solid depends on. + ## RDF-based Test Fixtures @@ -37,15 +38,13 @@ modules, but the most important scripts are currently in ## Test Suite Summary in Docker To summarize the test results, we have opted to build them in Docker, -and to run them within that. This also includes a fork of the W3C LDP -testsuite. This documents how to run it. +and to run them within that. This documents how to run it. Prerequisites: [Docker](https://docs.docker.com/install/) Run the following commands in your command line terminal: ```sh -docker build -t ldp-basic testers/ldp-basic docker build -t websockets-pubsub testers/websockets-pubsub docker build -t rdf-fixtures testers/rdf-fixtures @@ -64,12 +63,12 @@ egrep 'Tests:|tests run:|earl:outcome' reports/* | docker run -i table-reporter ``` The final output should look something like: ```sh -Server LDP Basic Websockets-pub-sub RDF-fixtures -gold 0/90 0/1 2/47 -inrupt-pod-server 0/90 0/1 2/47 -node-solid-server 15/90 0/1 22/49 -trellis 47/90 0/1 10/45 -wac-ldp 57/90 1/1 8/50 +Server Websockets-pub-sub RDF-fixtures +gold 0/1 2/47 +inrupt-pod-server 0/1 2/47 +node-solid-server 0/1 22/49 +trellis 0/1 10/45 +wac-ldp 1/1 8/50 ``` To run one tester against one server interactively, you can do for instance: @@ -81,18 +80,10 @@ root@f0c7e54fb1f3:/# exit [...] ``` -or: - -```sh -./runTesterAgainstServer.sh ldp-basic inrupt-pod-server -[...] -root@f0c7e54fb1f3:/# java -jar ldp-testsuite.jar --basic --server http://server:8080 --test PostContainer -root@f0c7e54fb1f3:/# exit -[...] -``` Caveat 1: the qualities of a software product can of course not be counted with a simple number of passing tests, so this list only gives a rough idea of levels of Solid spec compliance. Caveat 2: not all servers may have been configured optimally in these test runs; if you know of ways to improve the outcome for any of these servers, please provide a pull request. -Caveat 3: this test suite still incomplete and heavily biased towards LDP Basic container support; a lot of tests for WAC, websockets-pubsub, webid-oidc, sparql-get and globbing are still missing. +Caveat 3: this test suite still incomplete and development follows the +specification so it largely depends on rough consensus being reached there. diff --git a/reporters/table/report.js b/reporters/table/report.js index 8fb6995..5c100f6 100644 --- a/reporters/table/report.js +++ b/reporters/table/report.js @@ -1,13 +1,9 @@ -// reports/gold-ldp-basic.txt:Total tests run: 90, Failures: 20, Skips: 69 // reports/gold-websockets-pubsub.txt:Tests: 1 failed, 1 total // reports/gold.txt:Total tests run: 90, Failures: 20, Skips: 69 -// reports/inrupt-pod-server-ldp-basic.txt:Total tests run: 90, Failures: 16, Skips: 49 // reports/inrupt-pod-server-websockets-pubsub.txt:Tests: 1 passed, 1 total // reports/inrupt-pod-server.txt:Total tests run: 90, Failures: 16, Skips: 49 -// reports/node-solid-server-ldp-basic.txt:Total tests run: 90, Failures: 28, Skips: 47 // reports/node-solid-server-websockets-pubsub.txt:Tests: 1 failed, 1 total // reports/node-solid-server.txt:Total tests run: 90, Failures: 28, Skips: 47 -// reports/trellis-ldp-basic.txt:Total tests run: 90, Failures: 9, Skips: 29 // reports/trellis-websockets-pubsub.txt:Tests: 1 failed, 1 total // reports/trellis.txt:Total tests run: 90, Failures: 9, Skips: 29 @@ -58,33 +54,10 @@ function processWebsocketsPubsubLine (parts) { } table[serverName].websocketsPubsub = result; } -function processLdpBasicLine (parts) { - // console.log('process ldp-basic line', parts) - // reports/gold-ldp-basic.txt:Total tests run: 90, Failures: 20, Skips: 69 - // [ 'reports/gold-ldp-basic.txt:Total', 'tests', 'run:', - // '90,', 'Failures:', '20,', - // 'Skips:', '69' ] - const total = parseInt(parts[3]) - const failures = parseInt(parts[5]) - const skips = parseInt(parts[7]) - const result = `${total - failures - skips}/${total}` - // console.log(parts[3], total, parts[5], failures, parts[7], skips, result) - const serverName = parts[0].substring('reports/'.length, parts[0].length - '-ldp-basic.txt:Total'.length) - // console.log(parts[0], 'reports/'.length, parts[0].length, '-ldp-basic.txt:Total'.length, serverName) - if (!table[serverName]) { - table[serverName] = {}; - } - table[serverName].ldpBasic = result; -} function processLine (line) { const parts = line.split(' '); - if (line.indexOf('ldp-basic') !== -1) { - if (parts.length < 8) { - return; - } - processLdpBasicLine(parts) - } else if (line.indexOf('rdf-fixtures') !== -1) { + if (line.indexOf('rdf-fixtures') !== -1) { processPerlBasedLine(parts) } else { if (parts.length < 8) { @@ -95,11 +68,11 @@ function processLine (line) { } function writeOutput() { - console.log(['Server', 'LDP Basic', 'Websockets-pub-sub', 'RDF-fixtures'].map(str => str.padEnd(PAD_LEN)).join('\t')) + console.log(['Server', 'Websockets-pub-sub', 'RDF-fixtures'].map(str => str.padEnd(PAD_LEN)).join('\t')) for (let serverName in table) { // console.log(table[serverName], serverName) var perlBasedResult = `${table[serverName].perlBased.passedNumber}/${table[serverName].perlBased.totalNumber}` - console.log([serverName, table[serverName].ldpBasic, table[serverName].websocketsPubsub, perlBasedResult].map(str => str.padEnd(PAD_LEN)).join('\t')) + console.log([serverName, table[serverName].websocketsPubsub, perlBasedResult].map(str => str.padEnd(PAD_LEN)).join('\t')) } } diff --git a/runTests.sh b/runTests.sh index 833c3b9..6d12e22 100755 --- a/runTests.sh +++ b/runTests.sh @@ -13,9 +13,6 @@ if [[ "$1" == trellis ]] docker logs server fi -echo Running ldp-basic tester ... -docker run --network=testnet ldp-basic > reports/$1-ldp-basic.txt - echo Stopping server without WAC ... docker stop server diff --git a/testers/ldp-basic/.dockerignore b/testers/ldp-basic/.dockerignore deleted file mode 100644 index 9414382..0000000 --- a/testers/ldp-basic/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -Dockerfile diff --git a/testers/ldp-basic/Dockerfile b/testers/ldp-basic/Dockerfile deleted file mode 100644 index c949597..0000000 --- a/testers/ldp-basic/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM maven -RUN git clone https://github.com/trellis-ldp/ldp-testsuite -WORKDIR ldp-testsuite -RUN mvn install -DskipTests -Dgpg.skip -Dmaven.javadoc.skip -B -V -RUN mvn package -DskipTests -Dgpg.skip -Dmaven.javadoc.skip -B -V -RUN mvn verify -Dgpg.skip -Dmaven.javadoc.skip -RUN mkdir -p ldp && mv target/ldp-testsuite-0.2.0-SNAPSHOT-shaded.jar ldp/ldp-testsuite.jar -WORKDIR ldp -CMD java -jar ldp-testsuite.jar --basic --server http://server:8080