Merge pull request #382 from garywong-bc/master

Enabling S3 Documents

.gitignore

@@ -67,17 +67,18 @@ gwells.pyproj*
 /obj/
 /bin/
 
-#NavUnit
-navunit/.gradle/
-navunit/gebReports/
-
 #functional-tests
 functional-tests/.gradle/
 functional-tests/gebReports/
 functional-tests/build
+functional-tests/.project
+functional-tests/.settings/org.eclipse.buildship.core.prefs
+functional-tests/.classpath
+functional-tests/bin
 
 # Custom GW
 activate
 
 #virtual env
 /deactivate
+

Jenkinsfile-developer

@@ -71,6 +71,18 @@ node('bddstack') {
     }
 }
 
+stage('ZAP Security Scan') {
+    node('zap') {
+        stage('Scan Web Application') {
+            dir('/zap') {
+                def retVal = sh returnStatus: true, script: '/zap/zap-baseline.py -r baseline.html -t https://dlvrapps.nrs.gov.bc.ca/gwells'
+                publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, reportDir: '/zap/wrk', reportFiles: 'baseline.html', reportName: 'ZAP Baseline Scan', reportTitles: 'ZAP Baseline Scan'])
+                echo "Return value is: ${retVal}"
+            }
+        }
+    }
+}
+
 stage('Deploy on Test') {
     input "Deploy to test?"
     node('maven') {

README.md

@@ -20,6 +20,7 @@ Apart from the regular files created by Django (`project/*`, `welcome/*`, `manag
 ```
 database/           - Database-specific files
 └── code-tables     - Static code table sql scripts
+└── cron            - Shell scripts
 └── scripts         - PostgrSQL psql scripts
   └── sql-developer - SQL Developer Oracle SQL scripts
 
@@ -109,33 +110,48 @@ You can look at the combined stdout and stderr of a given pod with this command:
 
 This can be useful to observe the correct functioning of your application.
 
-
 ## Special environment variables
-
 Note that environment variables are case sensitive.
 
 ### APP_CONFIG
-
 You can fine tune the gunicorn configuration through the environment variable `APP_CONFIG` that, when set, should point to a config file as documented [here](http://docs.gunicorn.org/en/latest/settings.html).
 
-### DJANGO_SECRET_KEY
+### DB_REPLICATE 
+Until legacy WELLS is shutdown and all works done on GWELLS, there is a nightly replication of WELLS records to GWELLS Production.   This [variable](database/README.md#32) controls the behavior during deploys (to DEV/TEST/PROD) and is one of None, Subset, or Full.  
+
+* Recommended value on DEV:  `Subset` (otherwise the Functional Tests will fail)  
+* Recommended value on TEST: `Subset` or `Full`  
+* Recommended value on PROD: `Full`  
+
+### MINIO_ACCESS_KEY 
+Access key acting as a user ID that uniquely identifies the account.  Set as part of `gwells-minio` deployment but then used in `gwells` deployment to connect to the internal (private) Minio Server.
+
+### MINIO_SECRET_KEY 
+Secret key acting as the password to the account.  Set as part of `gwells-minio` deployment but then used in `gwells` deployment to connect to the internal (private) Minio Server.
+
+See our [Wiki page](https://github.com/bcgov/gwells/wiki/Storage-of-Related-Documents) for more details.
 
+### S3_HOST 
+Endpoint to the public S3 Server (e.g. `s3.ca-central-1.amazonaws.com`)
+
+### S3_ROOT_BUCKET 
+Top-level S3 bucket that organizes all publicly viewable documentes (e.g. `gwells-docs`)
+
+### DJANGO_SECRET_KEY
 When using one of the templates provided in this repository, this environment variable has its value automatically generated. For security purposes, make sure to set this to a random string as documented [here](https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SECRET_KEY).
 
 ### DJANGO_DEBUG 
-
 Set to `True` to enable debugging.  Recommended value:  `True`
 
-### ENABLE_DATA_ENTRY  
+NOTE: On local developer environments, the `gradlew` tests will fail with `DJANGO_DEBUG=False` unless the developer manually runs `python manage.py collectstatic`.
 
+### ENABLE_DATA_ENTRY  
 Set to `True` to enable debugging.  Recommended value:  Not set for Production (as the feature is not released); `True` for Development.
 
 ### ENABLE_GOOGLE_ANALYTICS
-
 Set to `True` to enable Google Analytics.  Recommended value:  Not set for Development; `True` for Production.
 
 ## One-off command execution
-
 At times you might want to manually execute some command in the context of a running application in OpenShift.
 You can drop into a Python shell for debugging, create a new user for the Django Admin interface, or perform any other task.
 

database/code-tables/registry/gwells_registry_appl_type.csv

@@ -0,0 +1,5 @@
+registry_appl_type_guid,code,subcode,description,is_hidden,sort_order,when_created,when_updated,who_created,who_updated
+"1f6c2aaa018c11e8ba890ed5f89f718b","DRILL","WATER","Water Well Driller","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c30c2018c11e8ba890ed5f89f718b","DRILL","GEOTECH","Geotechnical/Environmental Driller","N","4","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c3202018c11e8ba890ed5f89f718b","DRILL","GEOXCHG","Geoexchange Driller","N","6","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c2d0c018c11e8ba890ed5f89f718b","PUMP","PUMPINST",Pump Installer","N","8","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"

database/code-tables/registry/gwells_registry_qualified_for_types.csv

@@ -0,0 +1,12 @@
+registry_qual_for_guid,registry_appl_type_guid,code,description,is_hidden,sort_order,when_created,when_updated,who_created,who_updated
+"1f6c3324018c11e8ba890ed5f89f718b","1f6c2aaa018c11e8ba890ed5f89f718b","WAT","Water supply well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c3e46018c11e8ba890ed5f89f718b","1f6c2aaa018c11e8ba890ed5f89f718b","MON","Water supply well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c34aa018c11e8ba890ed5f89f718b","1f6c30c2018c11e8ba890ed5f89f718b","MON","Monitoring well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c35e0018c11e8ba890ed5f89f718b","1f6c2aaa018c11e8ba890ed5f89f718b","RECH","Recharge/Injection well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c372a018c11e8ba890ed5f89f718b","1f6c2aaa018c11e8ba890ed5f89f718b","DEWAT","Dewatering well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c3860018c11e8ba890ed5f89f718b","1f6c2aaa018c11e8ba890ed5f89f718b","REM","Remediatation well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c3f7c018c11e8ba890ed5f89f718b","1f6c30c2018c11e8ba890ed5f89f718b","REM","Remediatation well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c3bbc018c11e8ba890ed5f89f718b","1f6c2aaa018c11e8ba890ed5f89f718b","GEO","Geotechnical well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c4094018c11e8ba890ed5f89f718b","1f6c30c2018c11e8ba890ed5f89f718b","GEO","Geotechnical well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c3cde018c11e8ba890ed5f89f718b","1f6c3202018c11e8ba890ed5f89f718b","CLOS","Closed loop geoexchange well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"
+"1f6c41ac018c11e8ba890ed5f89f718b","1f6c2d0c018c11e8ba890ed5f89f718b","PUMP","Install pump in Water supply, Recharge/Injection, or Dewatering well","N","2","2018-01-01 00:00:00-08","2018-01-01 00:00:00-08","DATALOAD_USER","DATALOAD_USER"

functional-tests/src/test/groovy/Navigation/FlowSpecs.groovy

@@ -25,16 +25,16 @@ class FlowSpecs extends GebReportingSpec {
 			waitFor { $("a", id:"$clickLink").click() }
         then:
 			at assertPage
-		
+
         where:
         startPage                   | clickLink                | clickCount    | timeoutSeconds    || assertPage
         SearchPage                  | "ribbon-groundwaterinfo" | 1             | 3                 || AdditionalInformationPage
         AdditionalInformationPage   | "ribbon-search"          | 1             | 3                 || SearchPage
         SearchPage    		        | "ribbon-search"          | 1             | 3                 || SearchPage
         SearchPage                  | "BCWRAtlas"              | 1             | 3                 || BCWaterResourceAtlas
         SearchPage                  | "iMapBC"                 | 1             | 3                 || iMapBC
-        SearchPage                  | "ribbon-registry"        | 1             | 3                 || WellDrillerPage
-                
+        SearchPage                  | "ribbon-registry-legacy"        | 1             | 3                 || WellDrillerPage
+
         //Test Externally Linked Pages
 		SearchPage           | "footer-disclaimer"     | 1             | 3                 || Disclaimer
         SearchPage           | "footer-privacy"        | 1             | 3                 || Privacy

functional-tests/src/test/groovy/pages/app/WellDrillerPage..groovy → functional-tests/src/test/groovy/pages/app/WellDrillerPage.groovy

@@ -5,5 +5,5 @@ import geb.Page
 class WellDrillerPage extends Page {
     static at = { title == "Groundwater Wells Registry - Province of British Columbia" }
 
-    static url = "gwells/registry"
+    static url = "gwells/registry-legacy"
 }

gwells/minioClient.py

@@ -1,18 +1,16 @@
 import os
 from minio import Minio
 
-# NOTE 's3-ca-central-1.amazonaws.com' didn't work:
-# minio.error.ResponseError: ResponseError: code: AuthorizationHeaderMalformed, message: The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'ca-central-1', bucket_name: None, object_name: None, request_id: 3841822DCB7F7FC6, host_id: dk3GUWHqhX6gEiE6iONYAIfIpQNvbzExauHhCCoE4uTQOW6NygKY6yPdEsODXmChlO8Auk14ftg=, region:
+# NOTE Well Summary page uses publicly-viewable S3 bucket, so no credentials are needed
 #
 
 class MinioClient():
 
     def __init__(self):
-        self.minio_access_key = os.getenv('MINIO_ACCESS_KEY')
-        self.minio_secret_key = os.getenv('MINIO_SECRET_KEY')
+        self.access_key = ""
+        self.secret_key = ""
         self.host = os.getenv('S3_HOST')
-        self.link_host = os.getenv('S3_LINK_HOST')
-        self.minio_client = Minio(self.host, access_key=self.minio_access_key, secret_key=self.minio_secret_key, secure=True)
+        self.minio_client = Minio(self.host, access_key=self.access_key, secret_key=self.secret_key, secure=True)
         self.top_bucket = os.getenv('S3_ROOT_BUCKET')
 
     def get_documents(self, well_tag_number):

gwells/settings.py

@@ -76,6 +76,7 @@
     'gwells',
     'crispy_forms',
     'formtools',
+#     'registry',    
 )
 
 MIDDLEWARE = (

gwells/templates/gwells/base.html

@@ -136,7 +136,7 @@ <h1>Groundwater Wells and Aquifers</h1>
                         <ul class="nav navbar-nav">
                             <li><a id="navbar-search" href="{% url 'search' %}">Search</a></li>
                             <li><a id="navbar-groundwaterinfo" href="{% url 'groundwater_information' %}">GroundWater Information</a></li>
-                            <li><a id="ribbon-regdrop" href="{% url 'registry' %}">Registry</a></li>
+                            <li><a id="navbar-registry-legacy" href="{% url 'registry-legacy' %}">Registry</a></li>
                             {% if settings.ENABLE_DATA_ENTRY %}
                             <li><a id="navbar-submission" href="{% url 'activity_submission_create' %}">Submit Report</a></li>
                             {% endif %}
@@ -151,7 +151,7 @@ <h1>Groundwater Wells and Aquifers</h1>
                         <ul>
                             <li><a id="ribbon-search" href="{% url 'search' %}">Search</a></li>
                             <li><a id="ribbon-groundwaterinfo" href="{% url 'groundwater_information' %}">Groundwater Information</a></li>
-                            <li><a id="ribbon-registry" href="{% url 'registry' %}">Registry</a></li>
+                            <li><a id="ribbon-registry-legacy" href="{% url 'registry-legacy' %}">Registry</a></li>
                             {% if settings.ENABLE_DATA_ENTRY %}
                             <li><a id="navbar-submission" href="{% url 'activity_submission_create' %}">Submit Report</a></li>
                             {% endif %}
@@ -253,4 +253,4 @@ <h2>{% block bodyheading_block %}{% endblock %}</h2>
 
 </body>
 
-</html>
+</html>

gwells/templates/gwells/well_detail.html

@@ -868,7 +868,7 @@
                 <tbody>
                     {% for document in documents %}
                     <td>
-                        <a href="https://{{link_host}}/{{document.bucket_name}}/{{document.object_name}}" target="_blank">{{document.display_name}}</a>
+                        <a href="https://{{host}}/{{document.bucket_name}}/{{document.object_name}}" target="_blank">{{document.display_name}}</a>
                     </td>
                     {% endfor %}
                 </tbody>

gwells/test_views.py

@@ -85,7 +85,7 @@ def test_well_detail_ok(self):
         self.assertEqual(response.status_code, HTTPStatus.OK)
 
     def test_registry_ok(self):
-        self.ok('registry')
+        self.ok('registry-legacy')
 
     def test_health_ok(self):
         self.ok('health')

gwells/urls.py

@@ -34,13 +34,14 @@
     url(r'^'+ app_root_slash +'search$', SearchView.well_search, name='search'),
     # url(r'^(?P<pk>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/$', views.DetailView.as_view(), name='detail'),
     url(r'^'+ app_root_slash +'well/(?P<pk>[0-9]+)$', WellDetailView.as_view(), name='well_detail'),
-    url(r'^'+ app_root_slash +'registry$', RegistryView.as_view(), name='registry'),
+    url(r'^'+ app_root_slash +'registry-legacy$', RegistryView.as_view(), name='registry-legacy'),
     url(r'^'+ app_root_slash +'submission/(?P<pk>[0-9]+)$', ActivitySubmissionDetailView.as_view(), name='activity_submission_detail'),
     url(r'^'+ app_root_slash +'health$', HealthView.health, name='health'),
     url(r'^'+ app_root_slash +'groundwater-information', TemplateView.as_view(template_name='gwells/groundwater_information.html'), name='groundwater_information'),
     url(r'^'+ app_root_slash +'ajax/map_well_search/$', SearchView.map_well_search, name='map_well_search'),
     url(r'^'+ app_root_slash +'500.html$', TestErrorsView.test_500_view, name='test_500'),
     url(r'^'+ app_root_slash +'404.html$', TestErrorsView.test_404_view, name='test_404'),
+    url(r'^'+ app_root_slash +'registry/', include('registry.urls')),        
 ]
 
 if settings.ENABLE_DATA_ENTRY:

gwells/views/WellDetailView.py

@@ -24,7 +24,7 @@ def get_context_data(self, **kwargs):
 
                 minio_client = MinioClient()
 
-                context['link_host'] = minio_client.link_host;
+                context['host'] = minio_client.host;
                 context['documents'] = [];
 
                 documents = minio_client.get_documents(context['well'].well_tag_number)

registry/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

registry/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class RegistryConfig(AppConfig):
+    name = 'registry'