From cb360a2d0665ca957906395f6f81a8e9f44455c9 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 21:10:48 +1100 Subject: [PATCH 01/15] Increment version to 4.4.7. --- src/server/wsgi_version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/wsgi_version.h b/src/server/wsgi_version.h index 9539c039..4fa83c44 100644 --- a/src/server/wsgi_version.h +++ b/src/server/wsgi_version.h @@ -25,8 +25,8 @@ #define MOD_WSGI_MAJORVERSION_NUMBER 4 #define MOD_WSGI_MINORVERSION_NUMBER 4 -#define MOD_WSGI_MICROVERSION_NUMBER 6 -#define MOD_WSGI_VERSION_STRING "4.4.6" +#define MOD_WSGI_MICROVERSION_NUMBER 7 +#define MOD_WSGI_VERSION_STRING "4.4.7" /* ------------------------------------------------------------------------- */ From 3fc33bbcd50171c20aa8f9c0d6ca1c24aa5c7ff8 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 21:20:23 +1100 Subject: [PATCH 02/15] Trash dependency on mod_wsgi-metrics package and give up on trying to push people to actually try and use monitoring as way of making their setup run better. #65 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 6d86ba2e..9a2909b2 100644 --- a/setup.py +++ b/setup.py @@ -429,5 +429,4 @@ def _version(): ext_modules = [extension], entry_points = { 'console_scripts': ['mod_wsgi-express = mod_wsgi.server:main'],}, - install_requires = ['mod_wsgi-metrics >= 1.0.0'], ) From d6c250b5acab1278e0dfd2ebfc522ab81b88e130 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 21:25:15 +1100 Subject: [PATCH 03/15] Use environment variable to define the httpd modules directory so easy to load module from include Apache configuration file. --- src/server/__init__.py | 51 ++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/server/__init__.py b/src/server/__init__.py index 0209a977..078df563 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -77,7 +77,7 @@ def find_mimetypes(): APACHE_GENERAL_CONFIG = """ -LoadModule version_module '%(modules_directory)s/mod_version.so' +LoadModule version_module '${HTTPD_MODULES_DIRECTORY}/mod_version.so' ServerName %(host)s @@ -110,7 +110,7 @@ def find_mimetypes(): -LoadModule mpm_prefork_module '%(modules_directory)s/mod_mpm_prefork.so' +LoadModule mpm_prefork_module '${HTTPD_MODULES_DIRECTORY}/mod_mpm_prefork.so' @@ -123,13 +123,13 @@ def find_mimetypes(): -LoadModule mpm_event_module '%(modules_directory)s/mod_mpm_event.so' +LoadModule mpm_event_module '${HTTPD_MODULES_DIRECTORY}/mod_mpm_event.so' -LoadModule mpm_worker_module '%(modules_directory)s/mod_mpm_worker.so' +LoadModule mpm_worker_module '${HTTPD_MODULES_DIRECTORY}/mod_mpm_worker.so' -LoadModule mpm_prefork_module '%(modules_directory)s/mod_mpm_prefork.so' +LoadModule mpm_prefork_module '${HTTPD_MODULES_DIRECTORY}/mod_mpm_prefork.so' @@ -138,66 +138,66 @@ def find_mimetypes(): = 2.4> -LoadModule access_compat_module '%(modules_directory)s/mod_access_compat.so' +LoadModule access_compat_module '${HTTPD_MODULES_DIRECTORY}/mod_access_compat.so' -LoadModule unixd_module '%(modules_directory)s/mod_unixd.so' +LoadModule unixd_module '${HTTPD_MODULES_DIRECTORY}/mod_unixd.so' -LoadModule authn_core_module '%(modules_directory)s/mod_authn_core.so' +LoadModule authn_core_module '${HTTPD_MODULES_DIRECTORY}/mod_authn_core.so' -LoadModule authz_core_module '%(modules_directory)s/mod_authz_core.so' +LoadModule authz_core_module '${HTTPD_MODULES_DIRECTORY}/mod_authz_core.so' -LoadModule authz_host_module '%(modules_directory)s/mod_authz_host.so' +LoadModule authz_host_module '${HTTPD_MODULES_DIRECTORY}/mod_authz_host.so' -LoadModule mime_module '%(modules_directory)s/mod_mime.so' +LoadModule mime_module '${HTTPD_MODULES_DIRECTORY}/mod_mime.so' -LoadModule rewrite_module '%(modules_directory)s/mod_rewrite.so' +LoadModule rewrite_module '${HTTPD_MODULES_DIRECTORY}/mod_rewrite.so' -LoadModule alias_module '%(modules_directory)s/mod_alias.so' +LoadModule alias_module '${HTTPD_MODULES_DIRECTORY}/mod_alias.so' -LoadModule dir_module '%(modules_directory)s/mod_dir.so' +LoadModule dir_module '${HTTPD_MODULES_DIRECTORY}/mod_dir.so' -LoadModule env_module '%(modules_directory)s/mod_env.so' +LoadModule env_module '${HTTPD_MODULES_DIRECTORY}/mod_env.so' = 2.2.15> -LoadModule reqtimeout_module '%(modules_directory)s/mod_reqtimeout.so' +LoadModule reqtimeout_module '${HTTPD_MODULES_DIRECTORY}/mod_reqtimeout.so' -LoadModule deflate_module '%(modules_directory)s/mod_deflate.so' +LoadModule deflate_module '${HTTPD_MODULES_DIRECTORY}/mod_deflate.so' -LoadModule auth_basic_module '%(modules_directory)s/mod_auth_basic.so' +LoadModule auth_basic_module '${HTTPD_MODULES_DIRECTORY}/mod_auth_basic.so' -LoadModule auth_digest_module '%(modules_directory)s/mod_auth_digest.so' +LoadModule auth_digest_module '${HTTPD_MODULES_DIRECTORY}/mod_auth_digest.so' -LoadModule authz_user_module '%(modules_directory)s/mod_authz_user.so' +LoadModule authz_user_module '${HTTPD_MODULES_DIRECTORY}/mod_authz_user.so' -Loadmodule php5_module '%(modules_directory)s/libphp5.so' +Loadmodule php5_module '${HTTPD_MODULES_DIRECTORY}/libphp5.so' AddHandler application/x-httpd-php .php @@ -207,7 +207,7 @@ def find_mimetypes(): -LoadModule status_module '%(modules_directory)s/mod_status.so' +LoadModule status_module '${HTTPD_MODULES_DIRECTORY}/mod_status.so' @@ -340,7 +340,7 @@ def find_mimetypes(): -LoadModule log_config_module %(modules_directory)s/mod_log_config.so +LoadModule log_config_module ${HTTPD_MODULES_DIRECTORY}/mod_log_config.so LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b" common LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\"" combined @@ -360,7 +360,7 @@ def find_mimetypes(): -LoadModule ssl_module %(modules_directory)s/mod_ssl.so +LoadModule ssl_module ${HTTPD_MODULES_DIRECTORY}/mod_ssl.so @@ -1338,6 +1338,9 @@ def generate_server_metrics_script(options): HTTPD_COMMAND="$HTTPD $HTTPD_ARGS" +HTTPD_MODULES_DIRECTORY="%(modules_directory)s" +export HTTPD_MODULES_DIRECTORY + SHLIBPATH="%(shlibpath)s" if [ "x$SHLIBPATH" != "x" ]; then From 20d451d1850de95797680ba3f4426c88fbdd3b5a Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 21:30:02 +1100 Subject: [PATCH 04/15] Add release notes for 4.4.7. --- docs/release-notes/index.rst | 1 + docs/release-notes/version-4.4.7.rst | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 docs/release-notes/version-4.4.7.rst diff --git a/docs/release-notes/index.rst b/docs/release-notes/index.rst index e9cfb3e3..5726dadf 100644 --- a/docs/release-notes/index.rst +++ b/docs/release-notes/index.rst @@ -5,6 +5,7 @@ Release Notes .. toctree:: :maxdepth: 2 + version-4.4.7.rst version-4.4.6.rst version-4.4.5.rst version-4.4.4.rst diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst new file mode 100644 index 00000000..2c83c637 --- /dev/null +++ b/docs/release-notes/version-4.4.7.rst @@ -0,0 +1,11 @@ +============= +Version 4.4.7 +============= + +Version 4.4.7 of mod_wsgi can be obtained from: + + https://codeload.github.com/GrahamDumpleton/mod_wsgi/tar.gz/4.4.7 + +For details on the availability of Windows binaries see: + + https://github.com/GrahamDumpleton/mod_wsgi/tree/master/win32 From 71ea66d25df3a1ddea4a7927333377b2243078c8 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 21:30:52 +1100 Subject: [PATCH 05/15] Add --service-script option to mod_wsgi-express for starting up distinct managed applications. --- docs/release-notes/version-4.4.7.rst | 8 ++++++++ src/server/__init__.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index 2c83c637..be238d4b 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -9,3 +9,11 @@ Version 4.4.7 of mod_wsgi can be obtained from: For details on the availability of Windows binaries see: https://github.com/GrahamDumpleton/mod_wsgi/tree/master/win32 + +New Features +------------ + +1. Added ``--service-script`` option to ``mod_wsgi-express`` to allow a +Python script to be loaded and executed in the context of a distinct +daemon process. This can be used for executing a service to be managed by +Apache, even though it is a distinct application. diff --git a/src/server/__init__.py b/src/server/__init__.py index 078df563..b9088e07 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -674,6 +674,12 @@ def find_mimetypes(): process-group=express application-group=server-metrics """ +APACHE_SERVICE_CONFIG = """ +WSGIDaemonProcess 'service:%(name)s' display-name=%%{GROUP} threads=1 +WSGIImportScript '%(script)s' process-group='service:%(name)s' \\ + application-group=%%{GLOBAL} +""" + def generate_apache_config(options): with open(options['httpd_conf'], 'w') as fp: print(APACHE_GENERAL_CONFIG % options, file=fp) @@ -722,6 +728,11 @@ def generate_apache_config(options): print(APACHE_EXTENSION_CONFIG % dict(extension=extension), file=fp) + if options['service_scripts']: + for name, script in options['service_scripts']: + print(APACHE_SERVICE_CONFIG % dict(name=name, script=script), + file=fp) + if options['include_files']: for filename in options['include_files']: filename = os.path.abspath(filename) @@ -1912,6 +1923,12 @@ def check_percentage(option, opt_str, value, parser): optparse.make_option('--with-php5', action='store_true', default=False, help='Flag indicating whether PHP 5 support should be enabled.'), + optparse.make_option('--service-script', action='append', nargs=2, + dest='service_scripts', metavar='NAME SCRIPT-PATH', + help='Specify the name of a Python script to be loaded and ' + 'executed in the context of a distinct daemon process. Used ' + 'for running a managed service.'), + optparse.make_option('--enable-docs', action='store_true', default=False, help='Flag indicating whether the mod_wsgi documentation should ' 'be made available at the /__wsgi__/docs sub URL.'), @@ -2229,6 +2246,14 @@ def _cmd_setup_server(command, args, options): if options['with_newrelic_platform']: options['server_metrics'] = True + if options['service_scripts']: + service_scripts = [] + for name, script in options['service_scripts']: + if not os.path.isabs(script): + script = os.path.abspath(script) + service_scripts.append((name, script)) + options['service_scripts'] = service_scripts + max_clients = options['processes'] * options['threads'] if options['max_clients'] is not None: From 4070e0dce60c89da9c5b7f5852eb68160301f50e Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 21:48:34 +1100 Subject: [PATCH 06/15] Added --proxy-url-alias option to mod_wsgi-express for setting up proxying of sub URLs of the site to a remote URL. --- docs/release-notes/version-4.4.7.rst | 3 +++ src/server/__init__.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index be238d4b..903f1a08 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -17,3 +17,6 @@ New Features Python script to be loaded and executed in the context of a distinct daemon process. This can be used for executing a service to be managed by Apache, even though it is a distinct application. + +2. Added ``--proxy-url-alias`` option to ``mod_wsgi-express`` for setting +up proxying of a sub URL of the site to a remote URL. diff --git a/src/server/__init__.py b/src/server/__init__.py index b9088e07..66242e7b 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -194,6 +194,15 @@ def find_mimetypes(): + + +LoadModule proxy_module ${HTTPD_MODULES_DIRECTORY}/mod_proxy.so + + +LoadModule proxy_http_module ${HTTPD_MODULES_DIRECTORY}/mod_proxy_http.so + + + @@ -604,6 +613,10 @@ def find_mimetypes(): """ +APACHE_PROXY_PASS_CONFIG = """ +ProxyPass '%(mount_point)s' '%(url)s' +""" + APACHE_ALIAS_DIRECTORY_CONFIG = """ Alias '%(mount_point)s' '%(directory)s' @@ -684,6 +697,11 @@ def generate_apache_config(options): with open(options['httpd_conf'], 'w') as fp: print(APACHE_GENERAL_CONFIG % options, file=fp) + if options['proxy_url_aliases']: + for mount_point, url in options['proxy_url_aliases']: + print(APACHE_PROXY_PASS_CONFIG % dict( + mount_point=mount_point, url=url), file=fp) + if options['url_aliases']: for mount_point, target in sorted(options['url_aliases'], reverse=True): @@ -1730,6 +1748,11 @@ def check_percentage(option, opt_str, value, parser): default=False, help='Flag indicating whether Apache error ' 'documents will override application error responses.'), + optparse.make_option('--proxy-url-alias', action='append', nargs=2, + dest='proxy_url_aliases', metavar='URL-PATH URL', + help='Map a sub URL such that any requests against it will be ' + 'proxied to the specified URL.'), + optparse.make_option('--keep-alive-timeout', type='int', default=0, metavar='SECONDS', help='The number of seconds which a client ' 'connection will be kept alive to allow subsequent requests ' @@ -2519,6 +2542,8 @@ def _cmd_setup_server(command, args, options): options['httpd_arguments_list'].append('-DWSGI_CHUNKED_REQUEST') if options['with_php5']: options['httpd_arguments_list'].append('-DWSGI_WITH_PHP5') + if options['proxy_url_aliases']: + options['httpd_arguments_list'].append('-DWSGI_WITH_PROXY') options['httpd_arguments_list'].extend( _mpm_module_defines(options['modules_directory'], From 20605c0cf9f253d46c9210dc9ac903b0179336b7 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 22:19:47 +1100 Subject: [PATCH 07/15] Add --proxy-virtual-host option to mod_wsgi-express for setting up proxying of a whole virtual host. --- docs/release-notes/version-4.4.7.rst | 4 ++++ src/server/__init__.py | 27 ++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index 903f1a08..bf9c5381 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -20,3 +20,7 @@ Apache, even though it is a distinct application. 2. Added ``--proxy-url-alias`` option to ``mod_wsgi-express`` for setting up proxying of a sub URL of the site to a remote URL. + +3. Added ``--proxy-virtual-host`` option to ``mod_wsgi-express`` for setting +up proxying of a whole virtual host to a remote URL. Only supports proxying +of HTTP requests and not HTTPS requests. diff --git a/src/server/__init__.py b/src/server/__init__.py index 66242e7b..70fccad1 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -437,6 +437,14 @@ def find_mimetypes(): ThreadStackSize 262144 + + +NameVirtualHost *:%(port)s + + + + + @@ -617,6 +625,13 @@ def find_mimetypes(): ProxyPass '%(mount_point)s' '%(url)s' """ +APACHE_PROXY_PASS_HOST_CONFIG = """ + +ServerName %(host)s +ProxyPass / '%(url)s' + +""" + APACHE_ALIAS_DIRECTORY_CONFIG = """ Alias '%(mount_point)s' '%(directory)s' @@ -702,6 +717,12 @@ def generate_apache_config(options): print(APACHE_PROXY_PASS_CONFIG % dict( mount_point=mount_point, url=url), file=fp) + if options['proxy_virtual_hosts']: + for host, url in options['proxy_virtual_hosts']: + print(APACHE_PROXY_PASS_HOST_CONFIG % dict( + host=host, port=options['port'], url=url), + file=fp) + if options['url_aliases']: for mount_point, target in sorted(options['url_aliases'], reverse=True): @@ -1752,6 +1773,10 @@ def check_percentage(option, opt_str, value, parser): dest='proxy_url_aliases', metavar='URL-PATH URL', help='Map a sub URL such that any requests against it will be ' 'proxied to the specified URL.'), + optparse.make_option('--proxy-virtual-host', action='append', nargs=2, + dest='proxy_virtual_hosts', metavar='HOSTNAME URL', + help='Proxy any requests for the specified host name to the ' + 'remote URL.'), optparse.make_option('--keep-alive-timeout', type='int', default=0, metavar='SECONDS', help='The number of seconds which a client ' @@ -2542,7 +2567,7 @@ def _cmd_setup_server(command, args, options): options['httpd_arguments_list'].append('-DWSGI_CHUNKED_REQUEST') if options['with_php5']: options['httpd_arguments_list'].append('-DWSGI_WITH_PHP5') - if options['proxy_url_aliases']: + if options['proxy_url_aliases'] or options['proxy_virtual_hosts']: options['httpd_arguments_list'].append('-DWSGI_WITH_PROXY') options['httpd_arguments_list'].extend( From 2c727bc1ebe7ff6f2a02f54780c558defb534ca3 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sat, 24 Jan 2015 22:26:28 +1100 Subject: [PATCH 08/15] Renamed proxy-buffer-size option for WSGIDaemonProcess. --- docs/release-notes/version-4.4.7.rst | 8 ++++++++ src/server/__init__.py | 6 +++--- src/server/mod_wsgi.c | 16 ++++++++-------- src/server/wsgi_daemon.h | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index bf9c5381..cf9074ec 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -10,6 +10,14 @@ For details on the availability of Windows binaries see: https://github.com/GrahamDumpleton/mod_wsgi/tree/master/win32 +Features Changed +---------------- + +1. The ``proxy-buffer-size`` option to ``WSGIDaemonProcess`` directive +was renamed to ``response-buffer-size`` to avoid confusion with options +related to normal HTTP proxying. The ``--proxy-buffer-size`` option of +``mod_wsgi-express`` was similarly renamed to ``--response-buffer-size``. + New Features ------------ diff --git a/src/server/__init__.py b/src/server/__init__.py index 70fccad1..8189d7ee 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -271,7 +271,7 @@ def find_mimetypes(): send-buffer-size=%(send_buffer_size)s \\ receive-buffer-size=%(receive_buffer_size)s \\ header-buffer-size=%(header_buffer_size)s \\ - proxy-buffer-size=%(proxy_buffer_size)s \\ + response-buffer-size=%(response_buffer_size)s \\ server-metrics=%(daemon_server_metrics_flag)s @@ -294,7 +294,7 @@ def find_mimetypes(): shutdown-timeout=%(shutdown_timeout)s \\ send-buffer-size=%(send_buffer_size)s \\ receive-buffer-size=%(receive_buffer_size)s \\ - proxy-buffer-size=%(proxy_buffer_size)s \\ + response-buffer-size=%(response_buffer_size)s \\ server-metrics=%(daemon_server_metrics_flag)s @@ -1708,7 +1708,7 @@ def check_percentage(option, opt_str, value, parser): metavar='NUMBER', help='Size of buffer used for reading ' 'response headers from daemon processes. Defaults to 0, ' 'indicating internal default of 32768 bytes is used.'), - optparse.make_option('--proxy-buffer-size', type='int', default=0, + optparse.make_option('--response-buffer-size', type='int', default=0, metavar='NUMBER', help='Maximum amount of response content ' 'that will be allowed to be buffered in the Apache child ' 'worker process when proxying the response from a daemon ' diff --git a/src/server/mod_wsgi.c b/src/server/mod_wsgi.c index 96242772..f601cb8a 100644 --- a/src/server/mod_wsgi.c +++ b/src/server/mod_wsgi.c @@ -6609,7 +6609,7 @@ static const char *wsgi_add_daemon_process(cmd_parms *cmd, void *mconfig, int send_buffer_size = 0; int recv_buffer_size = 0; int header_buffer_size = 0; - int proxy_buffer_size = 0; + int response_buffer_size = 0; const char *script_user = NULL; const char *script_group = NULL; @@ -6873,13 +6873,13 @@ static const char *wsgi_add_daemon_process(cmd_parms *cmd, void *mconfig, "or 0 for default."; } } - else if (!strcmp(option, "proxy-buffer-size")) { + else if (!strcmp(option, "response-buffer-size")) { if (!*value) - return "Invalid proxy buffer size for WSGI daemon process."; + return "Invalid response buffer size for WSGI daemon process."; - proxy_buffer_size = atoi(value); - if (proxy_buffer_size < 65536 && proxy_buffer_size != 0) { - return "Proxy buffer size must be >= 65536 bytes, " + response_buffer_size = atoi(value); + if (response_buffer_size < 65536 && response_buffer_size != 0) { + return "Response buffer size must be >= 65536 bytes, " "or 0 for default."; } } @@ -7090,7 +7090,7 @@ static const char *wsgi_add_daemon_process(cmd_parms *cmd, void *mconfig, entry->send_buffer_size = send_buffer_size; entry->recv_buffer_size = recv_buffer_size; entry->header_buffer_size = header_buffer_size; - entry->proxy_buffer_size = proxy_buffer_size; + entry->response_buffer_size = response_buffer_size; entry->script_user = script_user; entry->script_group = script_group; @@ -11202,7 +11202,7 @@ static int wsgi_execute_remote(request_rec *r) /* Transfer any response content. */ - return wsgi_transfer_response(r, bbin, group->proxy_buffer_size); + return wsgi_transfer_response(r, bbin, group->response_buffer_size); } static apr_status_t wsgi_socket_read(apr_socket_t *sock, void *vbuf, diff --git a/src/server/wsgi_daemon.h b/src/server/wsgi_daemon.h index ff625e47..c6ba8a9a 100644 --- a/src/server/wsgi_daemon.h +++ b/src/server/wsgi_daemon.h @@ -122,7 +122,7 @@ typedef struct { int send_buffer_size; int recv_buffer_size; int header_buffer_size; - int proxy_buffer_size; + int response_buffer_size; const char *script_user; const char *script_group; int cpu_time_limit; From 8efb8120d502f8c4b9c6c972a89a2516200b7c00 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 25 Jan 2015 08:34:29 +1100 Subject: [PATCH 09/15] Added --service-user/--service-group options to mod_wsgi-express to allow user/group of service to be specified. --- docs/release-notes/version-4.4.7.rst | 9 +++++++- src/server/__init__.py | 33 ++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index cf9074ec..a9b8de27 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -24,7 +24,14 @@ New Features 1. Added ``--service-script`` option to ``mod_wsgi-express`` to allow a Python script to be loaded and executed in the context of a distinct daemon process. This can be used for executing a service to be managed by -Apache, even though it is a distinct application. +Apache, even though it is a distinct application. The options take two +arguments, a short name for the service and the path to the Python script +for starting the service. + +If ``mod_wsgi-express`` is being run as root, then a user and group can be +specified for the service using the ``--service-user`` and +``--service-group`` options. The options take two arguments, a short name +for the service and the user or group name respectively. 2. Added ``--proxy-url-alias`` option to ``mod_wsgi-express`` for setting up proxying of a sub URL of the site to a remote URL. diff --git a/src/server/__init__.py b/src/server/__init__.py index 8189d7ee..0f4fc271 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -703,7 +703,8 @@ def find_mimetypes(): """ APACHE_SERVICE_CONFIG = """ -WSGIDaemonProcess 'service:%(name)s' display-name=%%{GROUP} threads=1 +WSGIDaemonProcess 'service:%(name)s' display-name=%%{GROUP} threads=1 \\ + user='%(user)s' group='%(group)s' WSGIImportScript '%(script)s' process-group='service:%(name)s' \\ application-group=%%{GLOBAL} """ @@ -768,9 +769,13 @@ def generate_apache_config(options): file=fp) if options['service_scripts']: + users = dict(options['service_users'] or []) + groups = dict(options['service_groups'] or []) for name, script in options['service_scripts']: - print(APACHE_SERVICE_CONFIG % dict(name=name, script=script), - file=fp) + user = users.get(name, '${WSGI_RUN_USER}') + group = groups.get(name, '${WSGI_RUN_GROUP}') + print(APACHE_SERVICE_CONFIG % dict(name=name, user=user, + group=group, script=script), file=fp) if options['include_files']: for filename in options['include_files']: @@ -1723,12 +1728,12 @@ def check_percentage(option, opt_str, value, parser): 'the worker processes will still though be reloaded if the ' 'WSGI script file itself is modified.'), - optparse.make_option('--user', default=default_run_user(), metavar='NAME', - help='When being run by the root user, the user that the WSGI ' - 'application should be run as.'), + optparse.make_option('--user', default=default_run_user(), + metavar='USERNAME', help='When being run by the root user, ' + 'the user that the WSGI application should be run as.'), optparse.make_option('--group', default=default_run_group(), - metavar='NAME', help='When being run by the root user, the group ' - 'that the WSGI application should be run as.'), + metavar='GROUP', help='When being run by the root user, the ' + 'group that the WSGI application should be run as.'), optparse.make_option('--callable-object', default='application', metavar='NAME', help='The name of the entry point for the WSGI ' @@ -1972,10 +1977,20 @@ def check_percentage(option, opt_str, value, parser): help='Flag indicating whether PHP 5 support should be enabled.'), optparse.make_option('--service-script', action='append', nargs=2, - dest='service_scripts', metavar='NAME SCRIPT-PATH', + dest='service_scripts', metavar='SERVICE SCRIPT-PATH', help='Specify the name of a Python script to be loaded and ' 'executed in the context of a distinct daemon process. Used ' 'for running a managed service.'), + optparse.make_option('--service-user', action='append', nargs=2, + dest='service_users', metavar='SERVICE USERNAME', + help='When being run by the root user, the user that the ' + 'distinct daemon process started to run the managed service ' + 'should be run as.'), + optparse.make_option('--service-group', action='append', nargs=2, + dest='service_groups', metavar='SERVICE GROUP', + help='When being run by the root user, the group that the ' + 'distinct daemon process started to run the managed service ' + 'should be run as.'), optparse.make_option('--enable-docs', action='store_true', default=False, help='Flag indicating whether the mod_wsgi documentation should ' From 02ce3d0ddb786ac4da97667ef7378ecd7c1047dc Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 26 Jan 2015 16:40:43 +1100 Subject: [PATCH 10/15] Add eviction-timeout option to allow graceful period for process restart on signals to be set separately. --- docs/release-notes/version-4.4.7.rst | 18 ++++++++++++++++++ src/server/__init__.py | 16 ++++++++++++---- src/server/mod_wsgi.c | 24 +++++++++++++++++++++--- src/server/wsgi_daemon.h | 1 + 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index a9b8de27..346cf5ed 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -39,3 +39,21 @@ up proxying of a sub URL of the site to a remote URL. 3. Added ``--proxy-virtual-host`` option to ``mod_wsgi-express`` for setting up proxying of a whole virtual host to a remote URL. Only supports proxying of HTTP requests and not HTTPS requests. + +4. Added ``eviction-timeout`` option to ``WSGIDaemonProcess`` directive. +For the case where the graceful restart signal, usually ``SIGUSR1``, is +sent to a daemon process to evict the WSGI application and restart the +process, this controls how many seconds the process will wait, while still +accepting new requests, before it reaches an idle state with no active +requests and shuts down. + +The ``graceful-timeout`` option previously performed this exact role in +this case previously, but a separate option is being added to allow a +different timeout period to be specified for the case for forced eviction. +The existing ``graceful-timeout`` option is still used when a maximum +requests option or CPU usage limit is set. For backwards compatibility, +if ``eviction-timeout`` isn't set, it will fall back to using any value +specified using the ``graceful-timeout`` option. + +The ``--eviction-timeout`` option has also been added to +``mod_wsgi-express`` and behaves in a similar fashion. diff --git a/src/server/__init__.py b/src/server/__init__.py index 0f4fc271..2bb59ec0 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -267,6 +267,7 @@ def find_mimetypes(): inactivity-timeout=%(inactivity_timeout)s \\ deadlock-timeout=%(deadlock_timeout)s \\ graceful-timeout=%(graceful_timeout)s \\ + eviction-timeout=%(eviction_timeout)s \\ shutdown-timeout=%(shutdown_timeout)s \\ send-buffer-size=%(send_buffer_size)s \\ receive-buffer-size=%(receive_buffer_size)s \\ @@ -291,6 +292,7 @@ def find_mimetypes(): inactivity-timeout=%(inactivity_timeout)s \\ deadlock-timeout=%(deadlock_timeout)s \\ graceful-timeout=%(graceful_timeout)s \\ + eviction-timeout=%(eviction_timeout)s \\ shutdown-timeout=%(shutdown_timeout)s \\ send-buffer-size=%(send_buffer_size)s \\ receive-buffer-size=%(receive_buffer_size)s \\ @@ -1606,10 +1608,16 @@ def check_percentage(option, opt_str, value, parser): optparse.make_option('--graceful-timeout', type='int', default=15, metavar='SECONDS', help='Grace period for requests to complete ' - 'normally, without accepting new requests, when worker processes ' - 'are being shutdown and restarted due to maximum requests being ' - 'reached or due to graceful restart signal. Defaults to 15 ' - 'seconds.'), + 'normally, while still accepting new requests, when worker ' + 'processes are being shutdown and restarted due to maximum ' + 'requests being reached. Defaults to 15 seconds.'), + optparse.make_option('--eviction-timeout', type='int', default=0, + metavar='SECONDS', help='Grace period for requests to complete ' + 'normally, while still accepting new requests, when the WSGI ' + 'application is being evicted from the worker processes, and ' + 'the process restarted, due to forced graceful restart signal. ' + 'Defaults to timeout specified by \'--graceful-timeout\' ' + 'option.'), optparse.make_option('--deadlock-timeout', type='int', default=60, metavar='SECONDS', help='Maximum number of seconds allowed ' diff --git a/src/server/mod_wsgi.c b/src/server/mod_wsgi.c index f601cb8a..30ddbe5d 100644 --- a/src/server/mod_wsgi.c +++ b/src/server/mod_wsgi.c @@ -85,6 +85,7 @@ static apr_interval_time_t wsgi_deadlock_timeout = 0; static apr_interval_time_t wsgi_idle_timeout = 0; static apr_interval_time_t wsgi_request_timeout = 0; static apr_interval_time_t wsgi_graceful_timeout = 0; +static apr_interval_time_t wsgi_eviction_timeout = 0; static apr_time_t volatile wsgi_deadlock_shutdown_time = 0; static apr_time_t volatile wsgi_idle_shutdown_time = 0; static apr_time_t volatile wsgi_graceful_shutdown_time = 0; @@ -6596,6 +6597,7 @@ static const char *wsgi_add_daemon_process(cmd_parms *cmd, void *mconfig, int inactivity_timeout = 0; int request_timeout = 0; int graceful_timeout = 0; + int eviction_timeout = 0; int connect_timeout = 15; int socket_timeout = 0; int queue_timeout = 0; @@ -6808,6 +6810,14 @@ static const char *wsgi_add_daemon_process(cmd_parms *cmd, void *mconfig, if (graceful_timeout < 0) return "Invalid graceful timeout for WSGI daemon process."; } + else if (!strcmp(option, "eviction-timeout")) { + if (!*value) + return "Invalid eviction timeout for WSGI daemon process."; + + eviction_timeout = atoi(value); + if (eviction_timeout < 0) + return "Invalid eviction timeout for WSGI daemon process."; + } else if (!strcmp(option, "connect-timeout")) { if (!*value) return "Invalid connect timeout for WSGI daemon process."; @@ -7077,6 +7087,7 @@ static const char *wsgi_add_daemon_process(cmd_parms *cmd, void *mconfig, entry->inactivity_timeout = apr_time_from_sec(inactivity_timeout); entry->request_timeout = apr_time_from_sec(request_timeout); entry->graceful_timeout = apr_time_from_sec(graceful_timeout); + entry->eviction_timeout = apr_time_from_sec(eviction_timeout); entry->connect_timeout = apr_time_from_sec(connect_timeout); entry->socket_timeout = apr_time_from_sec(socket_timeout); entry->queue_timeout = apr_time_from_sec(queue_timeout); @@ -8288,6 +8299,9 @@ static void *wsgi_monitor_thread(apr_thread_t *thd, void *data) ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, wsgi_server, "mod_wsgi (pid=%d): Graceful timeout is %d.", getpid(), (int)(apr_time_sec(wsgi_graceful_timeout))); + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, wsgi_server, + "mod_wsgi (pid=%d): Eviction timeout is %d.", + getpid(), (int)(apr_time_sec(wsgi_eviction_timeout))); } while (1) { @@ -8579,6 +8593,7 @@ static void wsgi_daemon_main(apr_pool_t *p, WSGIDaemonProcess *daemon) wsgi_idle_timeout = daemon->group->inactivity_timeout; wsgi_request_timeout = daemon->group->request_timeout; wsgi_graceful_timeout = daemon->group->graceful_timeout; + wsgi_eviction_timeout = daemon->group->eviction_timeout; if (wsgi_deadlock_timeout || wsgi_idle_timeout) { rv = apr_thread_create(&reaper, thread_attr, wsgi_monitor_thread, @@ -8751,17 +8766,20 @@ static void wsgi_daemon_main(apr_pool_t *p, WSGIDaemonProcess *daemon) apr_thread_mutex_lock(wsgi_monitor_lock); wsgi_graceful_shutdown_time = apr_time_now(); - wsgi_graceful_shutdown_time += wsgi_graceful_timeout; + if (wsgi_eviction_timeout) + wsgi_graceful_shutdown_time += wsgi_eviction_timeout; + else + wsgi_graceful_shutdown_time += wsgi_graceful_timeout; apr_thread_mutex_unlock(wsgi_monitor_lock); ap_log_error(APLOG_MARK, APLOG_INFO, 0, wsgi_server, - "mod_wsgi (pid=%d): Graceful shutdown " + "mod_wsgi (pid=%d): Process eviction " "requested, waiting for requests to complete " "'%s'.", getpid(), daemon->group->name); } else { ap_log_error(APLOG_MARK, APLOG_INFO, 0, wsgi_server, - "mod_wsgi (pid=%d): Graceful shutdown " + "mod_wsgi (pid=%d): Process eviction " "requested, triggering immediate shutdown " "'%s'.", getpid(), daemon->group->name); diff --git a/src/server/wsgi_daemon.h b/src/server/wsgi_daemon.h index c6ba8a9a..999fca29 100644 --- a/src/server/wsgi_daemon.h +++ b/src/server/wsgi_daemon.h @@ -113,6 +113,7 @@ typedef struct { apr_time_t inactivity_timeout; apr_time_t request_timeout; apr_time_t graceful_timeout; + apr_time_t eviction_timeout; apr_time_t connect_timeout; apr_time_t socket_timeout; apr_time_t queue_timeout; From 520a0ae378ed679e93bc41965a59f4a54778a630 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 30 Jan 2015 21:41:42 +1100 Subject: [PATCH 11/15] Add ability to compile Apache from source code prior to building mod_wsgi itself. --- setup.py | 139 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 109 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index 9a2909b2..29743ada 100644 --- a/setup.py +++ b/setup.py @@ -19,43 +19,121 @@ from distutils.sysconfig import get_config_var as get_python_config from distutils.sysconfig import get_python_lib -# Before anything else, this setup.py uses various tricks to install -# precompiled Apache binaries for the Heroku and OpenShift environments. -# Once they are installed, then the installation of the mod_wsgi package -# itself will be triggered, ensuring that it can be built against the -# precompiled Apache binaries which were installed. -# -# We therefore first need to work out whether we are actually running on -# either Heroku of OpenShift. If we are, then we identify the set of -# precompiled binaries we are to use and copy it into the Python -# installation. +# Before anything else, this setup.py uses various tricks to potentially +# install Apache. This can be from source code if that ability is +# enabled, or from precompiled Apache binaries for Heroku and OpenShift +# environments. Once they are installed, then the installation of the +# mod_wsgi package itself will be triggered, ensuring that it can be +# built against the precompiled Apache binaries which were installed. + +def download_url(url): + package = os.path.basename(url) + if not os.path.isfile(package): + print('Downloading', url) + urlretrieve(url, package+'.download') + os.rename(package+'.download', package) + return package + +def extract_tar(src, dst): + print('Extracting', src) + tar = tarfile.open(src) + tar.extractall(dst) + tar.close() + +REQUIRE_APACHE = os.environ.get('MOD_WSGI_REQUIRE_APACHE') + +if REQUIRE_APACHE: + # If building from source code has been enabled, then we download + # the source code and compile it, then installing it where required. + + ASF_URL = 'http://www.us.apache.org/dist/' + + APR_URL = ASF_URL + 'apr/apr-1.5.1.tar.gz' + APR_UTIL_URL = ASF_URL + 'apr/apr-util-1.5.4.tar.gz' + APACHE_URL = ASF_URL + 'httpd/httpd-2.4.12.tar.gz' -PREFIX = 'https://s3.amazonaws.com' -BUCKET = os.environ.get('MOD_WSGI_REMOTE_S3_BUCKET_NAME', 'modwsgi.org') + download_url(APR_URL) + download_url(APR_UTIL_URL) + download_url(APACHE_URL) -REMOTE_TARBALL_NAME = os.environ.get('MOD_WSGI_REMOTE_PACKAGES_NAME') + if not os.path.isdir('build'): + os.mkdir('build') -TGZ_OPENSHIFT='mod_wsgi-packages-openshift-centos6-apache-2.4.10-1.tar.gz' -TGZ_HEROKU='mod_wsgi-packages-heroku-cedar14-apache-2.4.10-1.tar.gz' + if not os.path.isdir('build/packages'): + os.mkdir('build/packages') -if not REMOTE_TARBALL_NAME: - if os.environ.get('OPENSHIFT_HOMEDIR'): - REMOTE_TARBALL_NAME = TGZ_OPENSHIFT - elif os.path.isdir('/app/.heroku'): - REMOTE_TARBALL_NAME = TGZ_HEROKU + shutil.rmtree('build/apr-1.5.1', ignore_errors=True) + shutil.rmtree('build/apr-util-1.5.4', ignore_errors=True) + shutil.rmtree('build/httpd-2.4.12', ignore_errors=True) -LOCAL_TARBALL_FILE = os.environ.get('MOD_WSGI_LOCAL_PACKAGES_FILE') + extract_tar('apr-1.5.1.tar.gz', 'build') + extract_tar('apr-util-1.5.4.tar.gz', 'build') + extract_tar('httpd-2.4.12.tar.gz', 'build') -REMOTE_TARBALL_URL = None + shutil.rmtree('src/packages', ignore_errors=True) + + destdir = os.path.join(os.path.abspath( + os.path.dirname(__file__)), 'src/packages') + + if not os.path.isdir(destdir): + os.mkdir(destdir) -if LOCAL_TARBALL_FILE is None and REMOTE_TARBALL_NAME: - REMOTE_TARBALL_URL = '%s/%s/%s' % (PREFIX, BUCKET, REMOTE_TARBALL_NAME) + os.system('cd build/apr-1.5.1 && ' + './configure --prefix=%(destdir)s/apr && ' + 'make && make install' % dict(destdir=destdir)) -WITH_PACKAGES = False + os.system('cd build/apr-util-1.5.4 && ' + './configure --prefix=%(destdir)s/apr-util ' + '--with-apr=%(destdir)s/apr/bin/apr-1-config && ' + 'make && make install' % dict(destdir=destdir)) + + os.system('cd build/httpd-2.4.12 && ' + './configure --prefix=%(destdir)s/apache ' + '--enable-mpms-shared=all --enable-so --enable-rewrite ' + '--with-apr=%(destdir)s/apr/bin/apr-1-config ' + '--with-apr-util=%(destdir)s/apr-util/bin/apu-1-config && ' + 'make && make install' % dict(destdir=destdir)) + + shutil.rmtree('src/packages/apache/htdocs', ignore_errors=True) + shutil.rmtree('src/packages/apache/icons', ignore_errors=True) + shutil.rmtree('src/packages/apache/man', ignore_errors=True) + shutil.rmtree('src/packages/apache/manual', ignore_errors=True) -if REMOTE_TARBALL_URL or LOCAL_TARBALL_FILE: WITH_PACKAGES = True + REMOTE_TARBALL_URL = None + LOCAL_TARBALL_FILE = None + +else: + # Work out whether we are actually running on either Heroku of + # OpenShift. If we are, then we identify the set of precompiled + # binaries we are to use and copy it into the Python installation. + + PREFIX = 'https://s3.amazonaws.com' + BUCKET = os.environ.get('MOD_WSGI_REMOTE_S3_BUCKET_NAME', 'modwsgi.org') + + REMOTE_TARBALL_NAME = os.environ.get('MOD_WSGI_REMOTE_PACKAGES_NAME') + LOCAL_TARBALL_FILE = os.environ.get('MOD_WSGI_LOCAL_PACKAGES_FILE') + + TGZ_OPENSHIFT='mod_wsgi-packages-openshift-centos6-apache-2.4.10-1.tar.gz' + TGZ_HEROKU='mod_wsgi-packages-heroku-cedar14-apache-2.4.10-1.tar.gz' + + if not REMOTE_TARBALL_NAME and not LOCAL_TARBALL_FILE: + if os.environ.get('OPENSHIFT_HOMEDIR'): + REMOTE_TARBALL_NAME = TGZ_OPENSHIFT + elif os.path.isdir('/app/.heroku'): + REMOTE_TARBALL_NAME = TGZ_HEROKU + + REMOTE_TARBALL_URL = None + + if LOCAL_TARBALL_FILE is None and REMOTE_TARBALL_NAME: + REMOTE_TARBALL_URL = '%s/%s/%s' % (PREFIX, BUCKET, REMOTE_TARBALL_NAME) + + WITH_PACKAGES = False + + if REMOTE_TARBALL_URL or LOCAL_TARBALL_FILE: + WITH_PACKAGES = True + # If we are doing an install, download the tarball and unpack it into # the 'packages' subdirectory. We will then add everything in that # directory as package data so that it will be installed into the Python @@ -69,11 +147,12 @@ os.rename(REMOTE_TARBALL_NAME+'.download', REMOTE_TARBALL_NAME) LOCAL_TARBALL_FILE = REMOTE_TARBALL_NAME - shutil.rmtree('src/packages', ignore_errors=True) + if LOCAL_TARBALL_FILE: + shutil.rmtree('src/packages', ignore_errors=True) - tar = tarfile.open(LOCAL_TARBALL_FILE) - tar.extractall('src/packages') - tar.close() + tar = tarfile.open(LOCAL_TARBALL_FILE) + tar.extractall('src/packages') + tar.close() open('src/packages/__init__.py', 'a').close() From f1ef82155161744babea51aa0513537994e3a29d Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 30 Jan 2015 22:06:15 +1100 Subject: [PATCH 12/15] Raise an exception on a failed build of Apache from source code. --- setup.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 29743ada..3a2dac59 100644 --- a/setup.py +++ b/setup.py @@ -78,22 +78,31 @@ def extract_tar(src, dst): if not os.path.isdir(destdir): os.mkdir(destdir) - os.system('cd build/apr-1.5.1 && ' + res = os.system('cd build/apr-1.5.1 && ' './configure --prefix=%(destdir)s/apr && ' 'make && make install' % dict(destdir=destdir)) - os.system('cd build/apr-util-1.5.4 && ' + if res: + raise RuntimeError('Failed to build APR.') + + res = os.system('cd build/apr-util-1.5.4 && ' './configure --prefix=%(destdir)s/apr-util ' '--with-apr=%(destdir)s/apr/bin/apr-1-config && ' 'make && make install' % dict(destdir=destdir)) - os.system('cd build/httpd-2.4.12 && ' + if res: + raise RuntimeError('Failed to build APR-UTIL.') + + res = os.system('cd build/httpd-2.4.12 && ' './configure --prefix=%(destdir)s/apache ' '--enable-mpms-shared=all --enable-so --enable-rewrite ' '--with-apr=%(destdir)s/apr/bin/apr-1-config ' '--with-apr-util=%(destdir)s/apr-util/bin/apu-1-config && ' 'make && make install' % dict(destdir=destdir)) + if res: + raise RuntimeError('Failed to build APACHE.') + shutil.rmtree('src/packages/apache/htdocs', ignore_errors=True) shutil.rmtree('src/packages/apache/icons', ignore_errors=True) shutil.rmtree('src/packages/apache/man', ignore_errors=True) From 3ff9a3d42fbe68988b421fe94f18c6d633cb8ceb Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Sun, 1 Feb 2015 22:09:26 +1100 Subject: [PATCH 13/15] Move Apache httpd compilation from source code out into separate mod_wsgi-httpd package. --- setup.py | 175 +++++++++++++---------------------------- src/server/__init__.py | 9 +-- 2 files changed, 56 insertions(+), 128 deletions(-) diff --git a/setup.py b/setup.py index 3a2dac59..36bdd57d 100644 --- a/setup.py +++ b/setup.py @@ -19,136 +19,48 @@ from distutils.sysconfig import get_config_var as get_python_config from distutils.sysconfig import get_python_lib -# Before anything else, this setup.py uses various tricks to potentially -# install Apache. This can be from source code if that ability is -# enabled, or from precompiled Apache binaries for Heroku and OpenShift -# environments. Once they are installed, then the installation of the +# Before anything else, this setup.py uses some tricks to potentially +# install Apache. This can be from a local tarball, or from precompiled +# Apache binaries for Heroku and OpenShift environments downloaded from +# Amazon S3. Once they are installed, then the installation of the # mod_wsgi package itself will be triggered, ensuring that it can be # built against the precompiled Apache binaries which were installed. +# +# First work out whether we are actually running on either Heroku or +# OpenShift. If we are, then we identify the set of precompiled binaries +# we are to use and copy it into the Python installation. -def download_url(url): - package = os.path.basename(url) - if not os.path.isfile(package): - print('Downloading', url) - urlretrieve(url, package+'.download') - os.rename(package+'.download', package) - return package +PREFIX = 'https://s3.amazonaws.com' +BUCKET = os.environ.get('MOD_WSGI_REMOTE_S3_BUCKET_NAME', 'modwsgi.org') -def extract_tar(src, dst): - print('Extracting', src) - tar = tarfile.open(src) - tar.extractall(dst) - tar.close() +REMOTE_TARBALL_NAME = os.environ.get('MOD_WSGI_REMOTE_PACKAGES_NAME') +LOCAL_TARBALL_FILE = os.environ.get('MOD_WSGI_LOCAL_PACKAGES_FILE') -REQUIRE_APACHE = os.environ.get('MOD_WSGI_REQUIRE_APACHE') +TGZ_OPENSHIFT='mod_wsgi-packages-openshift-centos6-apache-2.4.10-1.tar.gz' +TGZ_HEROKU='mod_wsgi-packages-heroku-cedar14-apache-2.4.10-1.tar.gz' -if REQUIRE_APACHE: - # If building from source code has been enabled, then we download - # the source code and compile it, then installing it where required. +if not REMOTE_TARBALL_NAME and not LOCAL_TARBALL_FILE: + if os.environ.get('OPENSHIFT_HOMEDIR'): + REMOTE_TARBALL_NAME = TGZ_OPENSHIFT + elif os.path.isdir('/app/.heroku'): + REMOTE_TARBALL_NAME = TGZ_HEROKU - ASF_URL = 'http://www.us.apache.org/dist/' +REMOTE_TARBALL_URL = None - APR_URL = ASF_URL + 'apr/apr-1.5.1.tar.gz' - APR_UTIL_URL = ASF_URL + 'apr/apr-util-1.5.4.tar.gz' - APACHE_URL = ASF_URL + 'httpd/httpd-2.4.12.tar.gz' +if LOCAL_TARBALL_FILE is None and REMOTE_TARBALL_NAME: + REMOTE_TARBALL_URL = '%s/%s/%s' % (PREFIX, BUCKET, REMOTE_TARBALL_NAME) - download_url(APR_URL) - download_url(APR_UTIL_URL) - download_url(APACHE_URL) +WITH_TARBALL_PACKAGE = False - if not os.path.isdir('build'): - os.mkdir('build') - - if not os.path.isdir('build/packages'): - os.mkdir('build/packages') - - shutil.rmtree('build/apr-1.5.1', ignore_errors=True) - shutil.rmtree('build/apr-util-1.5.4', ignore_errors=True) - shutil.rmtree('build/httpd-2.4.12', ignore_errors=True) - - extract_tar('apr-1.5.1.tar.gz', 'build') - extract_tar('apr-util-1.5.4.tar.gz', 'build') - extract_tar('httpd-2.4.12.tar.gz', 'build') - - shutil.rmtree('src/packages', ignore_errors=True) - - destdir = os.path.join(os.path.abspath( - os.path.dirname(__file__)), 'src/packages') - - if not os.path.isdir(destdir): - os.mkdir(destdir) - - res = os.system('cd build/apr-1.5.1 && ' - './configure --prefix=%(destdir)s/apr && ' - 'make && make install' % dict(destdir=destdir)) - - if res: - raise RuntimeError('Failed to build APR.') - - res = os.system('cd build/apr-util-1.5.4 && ' - './configure --prefix=%(destdir)s/apr-util ' - '--with-apr=%(destdir)s/apr/bin/apr-1-config && ' - 'make && make install' % dict(destdir=destdir)) - - if res: - raise RuntimeError('Failed to build APR-UTIL.') - - res = os.system('cd build/httpd-2.4.12 && ' - './configure --prefix=%(destdir)s/apache ' - '--enable-mpms-shared=all --enable-so --enable-rewrite ' - '--with-apr=%(destdir)s/apr/bin/apr-1-config ' - '--with-apr-util=%(destdir)s/apr-util/bin/apu-1-config && ' - 'make && make install' % dict(destdir=destdir)) - - if res: - raise RuntimeError('Failed to build APACHE.') - - shutil.rmtree('src/packages/apache/htdocs', ignore_errors=True) - shutil.rmtree('src/packages/apache/icons', ignore_errors=True) - shutil.rmtree('src/packages/apache/man', ignore_errors=True) - shutil.rmtree('src/packages/apache/manual', ignore_errors=True) - - WITH_PACKAGES = True - - REMOTE_TARBALL_URL = None - LOCAL_TARBALL_FILE = None - -else: - # Work out whether we are actually running on either Heroku of - # OpenShift. If we are, then we identify the set of precompiled - # binaries we are to use and copy it into the Python installation. - - PREFIX = 'https://s3.amazonaws.com' - BUCKET = os.environ.get('MOD_WSGI_REMOTE_S3_BUCKET_NAME', 'modwsgi.org') - - REMOTE_TARBALL_NAME = os.environ.get('MOD_WSGI_REMOTE_PACKAGES_NAME') - LOCAL_TARBALL_FILE = os.environ.get('MOD_WSGI_LOCAL_PACKAGES_FILE') - - TGZ_OPENSHIFT='mod_wsgi-packages-openshift-centos6-apache-2.4.10-1.tar.gz' - TGZ_HEROKU='mod_wsgi-packages-heroku-cedar14-apache-2.4.10-1.tar.gz' - - if not REMOTE_TARBALL_NAME and not LOCAL_TARBALL_FILE: - if os.environ.get('OPENSHIFT_HOMEDIR'): - REMOTE_TARBALL_NAME = TGZ_OPENSHIFT - elif os.path.isdir('/app/.heroku'): - REMOTE_TARBALL_NAME = TGZ_HEROKU - - REMOTE_TARBALL_URL = None - - if LOCAL_TARBALL_FILE is None and REMOTE_TARBALL_NAME: - REMOTE_TARBALL_URL = '%s/%s/%s' % (PREFIX, BUCKET, REMOTE_TARBALL_NAME) - - WITH_PACKAGES = False - - if REMOTE_TARBALL_URL or LOCAL_TARBALL_FILE: - WITH_PACKAGES = True +if REMOTE_TARBALL_URL or LOCAL_TARBALL_FILE: + WITH_TARBALL_PACKAGE = True # If we are doing an install, download the tarball and unpack it into # the 'packages' subdirectory. We will then add everything in that # directory as package data so that it will be installed into the Python # installation. -if WITH_PACKAGES: +if WITH_TARBALL_PACKAGE: if REMOTE_TARBALL_URL: if not os.path.isfile(REMOTE_TARBALL_NAME): print('Downloading', REMOTE_TARBALL_URL) @@ -207,19 +119,27 @@ def find_program(names, default=None, paths=[]): APXS = os.environ.get('APXS') +WITH_HTTPD_PACKAGE = False + if APXS is None: - APXS = find_program(['apxs2', 'apxs'], 'apxs', ['/usr/sbin', os.getcwd()]) + APXS = find_program(['mod_wsgi-apxs']) + if APXS is not None: + WITH_HTTPD_PACKAGE = True + +if APXS is None: + APXS = find_program(['mod_wsgi-apxs', 'apxs2', 'apxs'], + 'apxs', ['/usr/sbin', os.getcwd()]) elif not os.path.isabs(APXS): APXS = find_program([APXS], APXS, ['/usr/sbin', os.getcwd()]) -if not WITH_PACKAGES: +if not WITH_TARBALL_PACKAGE: if not os.path.isabs(APXS) or not os.access(APXS, os.X_OK): raise RuntimeError('The %r command appears not to be installed or ' 'is not executable. Please check the list of prerequisites ' 'in the documentation for this package and install any ' 'missing Apache httpd server packages.' % APXS) -if WITH_PACKAGES: +if WITH_TARBALL_PACKAGE: SCRIPT_DIR = os.path.join(os.path.dirname(__file__), 'src', 'packages') CONFIG_FILE = os.path.join(SCRIPT_DIR, 'apache/build/config_vars.mk') @@ -336,18 +256,31 @@ def get_apxs_config(query): APXS_CONFIG_TEMPLATE = """ import os -WITH_PACKAGES = %(WITH_PACKAGES)r +WITH_TARBALL_PACKAGE = %(WITH_TARBALL_PACKAGE)r +WITH_HTTPD_PACKAGE = %(WITH_HTTPD_PACKAGE)r -if WITH_PACKAGES: +if WITH_HTTPD_PACKAGE: + import mod_wsgi.httpd + PACKAGES = os.path.join(os.path.dirname(mod_wsgi.httpd.__file__)) + BINDIR = os.path.join(PACKAGES, 'bin') + SBINDIR = BINDIR + LIBEXECDIR = os.path.join(PACKAGES, 'modules') + SHLIBPATH = os.path.join(PACKAGES, 'lib') +elif WITH_TARBALL_PACKAGE: import mod_wsgi.packages PACKAGES = os.path.join(os.path.dirname(mod_wsgi.packages.__file__)) BINDIR = os.path.join(PACKAGES, 'apache', 'bin') SBINDIR = BINDIR LIBEXECDIR = os.path.join(PACKAGES, 'apache', 'modules') + SHLIBPATH = [] + SHLIBPATH.append(os.path.join(PACKAGES, 'apr-util', 'lib')) + SHLIBPATH.append(os.path.join(PACKAGES, 'apr', 'lib')) + SHLIBPATH = ':'.join(SHLIBPATH) else: BINDIR = '%(BINDIR)s' SBINDIR = '%(SBINDIR)s' LIBEXECDIR = '%(LIBEXECDIR)s' + SHLIBPATH = '' MPM_NAME = '%(MPM_NAME)s' PROGNAME = '%(PROGNAME)s' @@ -370,7 +303,9 @@ def get_apxs_config(query): with open(os.path.join(os.path.dirname(__file__), 'src/server/apxs_config.py'), 'w') as fp: - print(APXS_CONFIG_TEMPLATE % dict(WITH_PACKAGES=WITH_PACKAGES, + print(APXS_CONFIG_TEMPLATE % dict( + WITH_TARBALL_PACKAGE=WITH_TARBALL_PACKAGE, + WITH_HTTPD_PACKAGE=WITH_HTTPD_PACKAGE, BINDIR=BINDIR, SBINDIR=SBINDIR, LIBEXECDIR=LIBEXECDIR, MPM_NAME=MPM_NAME, PROGNAME=PROGNAME, SHLIBPATH_VAR=SHLIBPATH_VAR), file=fp) diff --git a/src/server/__init__.py b/src/server/__init__.py index 2bb59ec0..63330246 100644 --- a/src/server/__init__.py +++ b/src/server/__init__.py @@ -2603,14 +2603,7 @@ def _cmd_setup_server(command, args, options): options['python_executable'] = sys.executable options['shlibpath_var'] = apxs_config.SHLIBPATH_VAR - - if apxs_config.WITH_PACKAGES: - shlibpath = [] - shlibpath.append(os.path.join(apxs_config.PACKAGES, 'apr-util', 'lib')) - shlibpath.append(os.path.join(apxs_config.PACKAGES, 'apr', 'lib')) - options['shlibpath'] = ':'.join(shlibpath) - else: - options['shlibpath'] = '' + options['shlibpath'] = apxs_config.SHLIBPATH generate_wsgi_handler_script(options) From 6cdc9e85681b1efe95987516ef57a323866da034 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 2 Feb 2015 14:14:32 +1100 Subject: [PATCH 14/15] Shouldn't be merging apxs config vars. --- setup.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/setup.py b/setup.py index 36bdd57d..b6b71dbd 100644 --- a/setup.py +++ b/setup.py @@ -197,29 +197,6 @@ def get_apxs_config(name): CONFIG['SBINDIR'] = get_apxs_config('sbindir') CONFIG['PROGNAME'] = get_apxs_config('progname') - _CFLAGS_NAMES = ['SHLTCFLAGS', 'CFLAGS', 'NOTEST_CPPFLAGS', - 'EXTRA_CPPFLAGS', 'EXTRA_CFLAGS'] - - _CFLAGS_VALUES = [] - - for name in _CFLAGS_NAMES: - value = get_apxs_config(name) - - # Heroku doesn't appear to run the same version of gcc - # that a standard Ubuntu installation does and which was - # used to originally build the Apache binaries. We need - # therefore to strip out flags that the Heroku gcc may - # not understand. - - if value: - if os.path.isdir('/app/.heroku'): - value = value.replace('-prefer-pic', '') - - if value: - _CFLAGS_VALUES.append(value) - - CONFIG['CFLAGS'] = ' '.join(_CFLAGS_VALUES) - else: def get_apxs_config(query): p = subprocess.Popen([APXS, '-q', query], From b6f1291009109d9bb66330d16179b05680908469 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Mon, 2 Feb 2015 14:25:34 +1100 Subject: [PATCH 15/15] Document support for mod_wsgi-httpd package. --- docs/release-notes/version-4.4.7.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/release-notes/version-4.4.7.rst b/docs/release-notes/version-4.4.7.rst index 346cf5ed..a1424386 100644 --- a/docs/release-notes/version-4.4.7.rst +++ b/docs/release-notes/version-4.4.7.rst @@ -57,3 +57,13 @@ specified using the ``graceful-timeout`` option. The ``--eviction-timeout`` option has also been added to ``mod_wsgi-express`` and behaves in a similar fashion. + +5. Added support for new ``mod_wsgi-httpd`` package. The ``mod_wsgi-httpd`` +package is a pip installable package which will build the Apache httpd +server and install it into the Python installation. If the +``mod_wsgi-httpd`` package is installed before installing this package, +then the Apache httpd server installation installed by ``mod_wsgi-httpd`` +will be used instead of any system installed version of the Apache httpd +server when running ``mod_wsgi-express``. This allows you to workaround +any inability to upgrade the main Apache installation, or install its 'dev' +package if missing, or install it outright if not present.