From 94a785320226f197b4237145c35d3425befdc0ab Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 29 Oct 2024 10:00:35 +0100 Subject: [PATCH] internal: move virtuoso.ini files to seperate config folder --- .changeset/witty-paws-taste.md | 9 + .gitignore | 1 - config/virtuoso/virtuoso-production.ini | 249 ++++++++++++++++++++++ config/virtuoso/virtuoso-qa.ini | 249 ++++++++++++++++++++++ {data/db => config/virtuoso}/virtuoso.ini | 12 +- docker-compose.yml | 1 + 6 files changed, 514 insertions(+), 7 deletions(-) create mode 100644 .changeset/witty-paws-taste.md create mode 100644 config/virtuoso/virtuoso-production.ini create mode 100644 config/virtuoso/virtuoso-qa.ini rename {data/db => config/virtuoso}/virtuoso.ini (96%) diff --git a/.changeset/witty-paws-taste.md b/.changeset/witty-paws-taste.md new file mode 100644 index 00000000..2a15cf54 --- /dev/null +++ b/.changeset/witty-paws-taste.md @@ -0,0 +1,9 @@ +--- +"app-gelinkt-notuleren": patch +--- + +Move `virtuoso.ini` files to seperate folder +Introduce seperate `virtuoso.ini` files for: +- Local use +- DEV/QA +- Production diff --git a/.gitignore b/.gitignore index 005458e8..a26d0305 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ /data/db/virtuoso.log /data/db/virtuoso.pxa /data/db/virtuoso.trx -!/data/db/virtuoso.ini # Delta files /data/files diff --git a/config/virtuoso/virtuoso-production.ini b/config/virtuoso/virtuoso-production.ini new file mode 100644 index 00000000..aef3a9f7 --- /dev/null +++ b/config/virtuoso/virtuoso-production.ini @@ -0,0 +1,249 @@ +; +; virtuoso.ini +; +; Configuration file for the OpenLink Virtuoso VDBMS Server +; +; To learn more about this product, or any other product in our +; portfolio, please check out our web site at: +; +; http://virtuoso.openlinksw.com/ +; +; or contact us at: +; +; general.information@openlinksw.com +; +; If you have any technical questions, please contact our support +; staff at: +; +; technical.support@openlinksw.com +; +; +; Database setup +; +[Database] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db +ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log +LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx +xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa +ErrorLogLevel = 7 +FileExtend = 200 +MaxCheckpointRemap = 20000 +Striping = 0 +TempStorage = TempDatabase +Syslog = 0 ; write to syslog in case of failures where writing to log is not possible + +[TempDatabase] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx +MaxCheckpointRemap = 000 +Striping = 0 + +; +; Server parameters +; +[Parameters] +ServerPort = 1111 +LiteMode = 0 +DisableUnixSocket = 1 +DisableTcpSocket = 0 +;SSLServerPort = 2111 +;SSLCertificate = cert.pem +;SSLPrivateKey = pk.pem +;X509ClientVerify = 0 +;X509ClientVerifyDepth = 0 +;X509ClientVerifyCAFile = ca.pem +MaxClientConnections = 10 +CheckpointInterval = 60 +TraceOn = errors ; write traces +O_DIRECT = 0 +CaseMode = 2 +MaxStaticCursorRows = 5000 +CheckpointAuditTrail = 0 +AllowOSCalls = 0 +SchedulerInterval = 10 +DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad +ThreadCleanupInterval = 10 +ThreadThreshold = 10 +ResourcesCleanupInterval = 10 +FreeTextBatchSize = 100000 +SingleCPU = 0 +VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/ +PrefixResultNames = 0 +RdfFreeTextRulesSize = 100 +IndexTreeMaps = 256 +MaxMemPoolSize = 200000000 +PrefixResultNames = 0 +MacSpotlight = 0 +MaxQueryMem = 8G ; memory allocated to query processor ;;Niels said yeah let's test: https://docs.openlinksw.com/virtuoso/ptune/ +VectorSize = 1000 ; initial parallel query vector (array of query operations) size +MaxVectorSize = 1000000 ; query vector size threshold. +AdjustVectorSize = 1 +ThreadsPerQuery = 4 +AsyncQueueMaxThreads = 10 +TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back) +;; +;; When running with large data sets, one should configure the Virtuoso +;; process to use between 2/3 to 3/5 of free system memory and to stripe +;; storage on all available disks. +;; +;; Uncomment next two lines if there is 2 GB system memory free +; NumberOfBuffers = 170000 +; MaxDirtyBuffers = 130000 + +;; uncomment next two lines if there is 4 GB system memory free +;NumberOfBuffers = 340000 +;MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 8 GB system memory free +NumberOfBuffers = 680000 +MaxDirtyBuffers = 500000 +;; Uncomment next two lines if there is 16 GB system memory free +;NumberOfBuffers = 1360000 +;MaxDirtyBuffers = 1000000 +;; Uncomment next two lines if there is 32 GB system memory free +;NumberOfBuffers = 2720000 +;MaxDirtyBuffers = 2000000 +;; Uncomment next two lines if there is 48 GB system memory free +;NumberOfBuffers = 4000000 +;MaxDirtyBuffers = 3000000 +;; Uncomment next two lines if there is 64 GB system memory free +;NumberOfBuffers = 5450000 +;MaxDirtyBuffers = 4000000 +;; +;; Note the default settings will take very little memory +;; but will not result in very good performance +;; +;NumberOfBuffers = 10000 +;MaxDirtyBuffers = 6000 +[HTTPServer] +ServerPort = 8890 +ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp +MaxClientConnections = 10 +DavRoot = DAV +EnabledDavVSP = 0 +HTTPProxyEnabled = 0 +TempASPXDir = 0 +DefaultMailServer = localhost:25 +ServerThreads = 10 +MaxKeepAlives = 10 +KeepAliveTimeout = 10 +MaxCachedProxyConnections = 10 +ProxyConnectionCacheTimeout = 15 +HTTPThreadSize = 280000 +HttpPrintWarningsInOutput = 0 +Charset = UTF-8 +;;HTTPLogFile = http.log +MaintenancePage = atomic.html +EnabledGzipContent = 1 + +[AutoRepair] +BadParentLinks = 0 + +[Client] +SQL_PREFETCH_ROWS = 100 +SQL_PREFETCH_BYTES = 16000 +SQL_QUERY_TIMEOUT = 0 +SQL_TXN_TIMEOUT = 0 +;SQL_NO_CHAR_C_ESCAPE = 1 +;SQL_UTF8_EXECS = 0 +;SQL_NO_SYSTEM_TABLES = 0 +;SQL_BINARY_TIMESTAMP = 1 +;SQL_ENCRYPTION_ON_PASSWORD = -1 + +[VDB] +ArrayOptimization = 0 +NumArrayParameters = 10 +VDBDisconnectTimeout = 1000 +KeepConnectionOnFixedThread = 0 + +[Replication] +ServerName = db-D602566B774E +ServerEnable = 1 +QueueMax = 50000 + +; +; Striping setup +; +; These parameters have only effect when Striping is set to 1 in the +; [Database] section, in which case the DatabaseFile parameter is ignored. +; +; With striping, the database is spawned across multiple segments +; where each segment can have multiple stripes. +; +; Format of the lines below: +; Segment = , [, .. ] +; +; must be ordered from 1 up. +; +; The is the total size of the segment which is equally divided +; across all stripes forming the segment. Its specification can be in +; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks +; (b, the default) +; +; Note that the segment size must be a multiple of the database page size +; which is currently 8k. Also, the segment size must be divisible by the +; number of stripe files forming the segment. +; +; The example below creates a 200 meg database striped on two segments +; with two stripes of 50 meg and one of 100 meg. +; +; You can always add more segments to the configuration, but once +; added, do not change the setup. +; +[Striping] +Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +Segment2 = 100M, db-seg2-1.db +;... +;[TempStriping] +;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +;Segment2 = 100M, db-seg2-1.db +;... +;[Ucms] +;UcmPath = +;Ucm1 = +;Ucm2 = +;... + +[Zero Config] +ServerName = virtuoso (D602566B774E) +;ServerDSN = ZDSN +;SSLServerName = +;SSLServerDSN = + +[Mono] +;MONO_TRACE = Off +;MONO_PATH = +;MONO_ROOT = +;MONO_CFG_DIR = +;virtclr.dll = + +[URIQA] +DynamicLocal = 0 +DefaultHost = localhost:8890 + +[SPARQL] +;ExternalQuerySource = 1 +;ExternalXsltSource = 1 +;DefaultGraph = http://localhost:8890/dataspace +;ImmutableGraphs = http://localhost:8890/dataspace +ResultSetMaxRows = 10000 +;MaxQueryCostEstimationTime = 4000 ; in seconds +MaxQueryExecutionTime = 600 ; in seconds +DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100 +DeferInferenceRulesInit = 0 ; controls inference rules loading +;PingService = http://rpc.pingthesemanticweb.com/ + +[Plugins] +LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting +;Load1 = plain, wikiv +;Load2 = plain, mediawiki +;Load3 = plain, creolewiki +;Load4 = plain, im +;Load5 = plain, wbxml2 +;Load6 = plain, hslookup +;Load7 = attach, libphp5.so +;Load8 = Hosting, hosting_php.so +;Load9 = Hosting,hosting_perl.so +;Load10 = Hosting,hosting_python.so +;Load11 = Hosting,hosting_ruby.so +;Load12 = msdtc,msdtc_sample \ No newline at end of file diff --git a/config/virtuoso/virtuoso-qa.ini b/config/virtuoso/virtuoso-qa.ini new file mode 100644 index 00000000..61cc2662 --- /dev/null +++ b/config/virtuoso/virtuoso-qa.ini @@ -0,0 +1,249 @@ +; +; virtuoso.ini +; +; Configuration file for the OpenLink Virtuoso VDBMS Server +; +; To learn more about this product, or any other product in our +; portfolio, please check out our web site at: +; +; http://virtuoso.openlinksw.com/ +; +; or contact us at: +; +; general.information@openlinksw.com +; +; If you have any technical questions, please contact our support +; staff at: +; +; technical.support@openlinksw.com +; +; +; Database setup +; +[Database] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db +ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log +LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx +xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa +ErrorLogLevel = 7 +FileExtend = 200 +MaxCheckpointRemap = 2000 +Striping = 0 +TempStorage = TempDatabase +Syslog = 1 ; write to syslog in case of failures where writing to log is not possible + +[TempDatabase] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx +MaxCheckpointRemap = 2000 +Striping = 0 + +; +; Server parameters +; +[Parameters] +ServerPort = 1111 +LiteMode = 0 +DisableUnixSocket = 1 +DisableTcpSocket = 0 +;SSLServerPort = 2111 +;SSLCertificate = cert.pem +;SSLPrivateKey = pk.pem +;X509ClientVerify = 0 +;X509ClientVerifyDepth = 0 +;X509ClientVerifyCAFile = ca.pem +MaxClientConnections = 10 +CheckpointInterval = 60 +TraceOn = errors ; write traces +O_DIRECT = 0 +CaseMode = 2 +MaxStaticCursorRows = 5000 +CheckpointAuditTrail = 0 +AllowOSCalls = 0 +SchedulerInterval = 10 +DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad +ThreadCleanupInterval = 10 +ThreadThreshold = 10 +ResourcesCleanupInterval = 10 +FreeTextBatchSize = 100000 +SingleCPU = 0 +VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/ +PrefixResultNames = 0 +RdfFreeTextRulesSize = 100 +IndexTreeMaps = 256 +MaxMemPoolSize = 200000000 +PrefixResultNames = 0 +MacSpotlight = 0 +MaxQueryMem = 4G ; memory allocated to query processor ;;Niels said to 4g +VectorSize = 1000 ; initial parallel query vector (array of query operations) size +MaxVectorSize = 1000000 ; query vector size threshold. +AdjustVectorSize = 0 +ThreadsPerQuery = 4 +AsyncQueueMaxThreads = 10 +TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back) +;; +;; When running with large data sets, one should configure the Virtuoso +;; process to use between 2/3 to 3/5 of free system memory and to stripe +;; storage on all available disks. +;; +;; Uncomment next two lines if there is 2 GB system memory free +; NumberOfBuffers = 170000 +; MaxDirtyBuffers = 130000 + +;; Uncomment next two lines if there is 4 GB system memory free +;NumberOfBuffers = 340000 +;MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 8 GB system memory free +NumberOfBuffers = 680000 +MaxDirtyBuffers = 500000 +;; Uncomment next two lines if there is 16 GB system memory free +;NumberOfBuffers = 1360000 +;MaxDirtyBuffers = 1000000 +;; Uncomment next two lines if there is 32 GB system memory free +;NumberOfBuffers = 2720000 +;MaxDirtyBuffers = 2000000 +;; Uncomment next two lines if there is 48 GB system memory free +;NumberOfBuffers = 4000000 +;MaxDirtyBuffers = 3000000 +;; Uncomment next two lines if there is 64 GB system memory free +;NumberOfBuffers = 5450000 +;MaxDirtyBuffers = 4000000 +;; +;; Note the default settings will take very little memory +;; but will not result in very good performance +;; +;NumberOfBuffers = 10000 +;MaxDirtyBuffers = 6000 +[HTTPServer] +ServerPort = 8890 +ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp +MaxClientConnections = 10 +DavRoot = DAV +EnabledDavVSP = 0 +HTTPProxyEnabled = 0 +TempASPXDir = 0 +DefaultMailServer = localhost:25 +ServerThreads = 10 +MaxKeepAlives = 10 +KeepAliveTimeout = 10 +MaxCachedProxyConnections = 10 +ProxyConnectionCacheTimeout = 15 +HTTPThreadSize = 280000 +HttpPrintWarningsInOutput = 0 +Charset = UTF-8 +;;HTTPLogFile = http.log +MaintenancePage = atomic.html +EnabledGzipContent = 1 + +[AutoRepair] +BadParentLinks = 0 + +[Client] +SQL_PREFETCH_ROWS = 100 +SQL_PREFETCH_BYTES = 16000 +SQL_QUERY_TIMEOUT = 0 +SQL_TXN_TIMEOUT = 0 +;SQL_NO_CHAR_C_ESCAPE = 1 +;SQL_UTF8_EXECS = 0 +;SQL_NO_SYSTEM_TABLES = 0 +;SQL_BINARY_TIMESTAMP = 1 +;SQL_ENCRYPTION_ON_PASSWORD = -1 + +[VDB] +ArrayOptimization = 0 +NumArrayParameters = 10 +VDBDisconnectTimeout = 1000 +KeepConnectionOnFixedThread = 0 + +[Replication] +ServerName = db-D602566B774E +ServerEnable = 1 +QueueMax = 50000 + +; +; Striping setup +; +; These parameters have only effect when Striping is set to 1 in the +; [Database] section, in which case the DatabaseFile parameter is ignored. +; +; With striping, the database is spawned across multiple segments +; where each segment can have multiple stripes. +; +; Format of the lines below: +; Segment = , [, .. ] +; +; must be ordered from 1 up. +; +; The is the total size of the segment which is equally divided +; across all stripes forming the segment. Its specification can be in +; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks +; (b, the default) +; +; Note that the segment size must be a multiple of the database page size +; which is currently 8k. Also, the segment size must be divisible by the +; number of stripe files forming the segment. +; +; The example below creates a 200 meg database striped on two segments +; with two stripes of 50 meg and one of 100 meg. +; +; You can always add more segments to the configuration, but once +; added, do not change the setup. +; +[Striping] +Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +Segment2 = 100M, db-seg2-1.db +;... +;[TempStriping] +;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +;Segment2 = 100M, db-seg2-1.db +;... +;[Ucms] +;UcmPath = +;Ucm1 = +;Ucm2 = +;... + +[Zero Config] +ServerName = virtuoso (D602566B774E) +;ServerDSN = ZDSN +;SSLServerName = +;SSLServerDSN = + +[Mono] +;MONO_TRACE = Off +;MONO_PATH = +;MONO_ROOT = +;MONO_CFG_DIR = +;virtclr.dll = + +[URIQA] +DynamicLocal = 0 +DefaultHost = localhost:8890 + +[SPARQL] +;ExternalQuerySource = 1 +;ExternalXsltSource = 1 +;DefaultGraph = http://localhost:8890/dataspace +;ImmutableGraphs = http://localhost:8890/dataspace +ResultSetMaxRows = 10000 +;MaxQueryCostEstimationTime = 4000 ; in seconds +MaxQueryExecutionTime = 600 ; in seconds +DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100 +DeferInferenceRulesInit = 0 ; controls inference rules loading +;PingService = http://rpc.pingthesemanticweb.com/ + +[Plugins] +LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting +;Load1 = plain, wikiv +;Load2 = plain, mediawiki +;Load3 = plain, creolewiki +;Load4 = plain, im +;Load5 = plain, wbxml2 +;Load6 = plain, hslookup +;Load7 = attach, libphp5.so +;Load8 = Hosting, hosting_php.so +;Load9 = Hosting,hosting_perl.so +;Load10 = Hosting,hosting_python.so +;Load11 = Hosting,hosting_ruby.so +;Load12 = msdtc,msdtc_sample diff --git a/data/db/virtuoso.ini b/config/virtuoso/virtuoso.ini similarity index 96% rename from data/db/virtuoso.ini rename to config/virtuoso/virtuoso.ini index 118bb205..61cc2662 100644 --- a/data/db/virtuoso.ini +++ b/config/virtuoso/virtuoso.ini @@ -75,7 +75,7 @@ IndexTreeMaps = 256 MaxMemPoolSize = 200000000 PrefixResultNames = 0 MacSpotlight = 0 -MaxQueryMem = 2G ; memory allocated to query processor +MaxQueryMem = 4G ; memory allocated to query processor ;;Niels said to 4g VectorSize = 1000 ; initial parallel query vector (array of query operations) size MaxVectorSize = 1000000 ; query vector size threshold. AdjustVectorSize = 0 @@ -91,12 +91,12 @@ TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient ; NumberOfBuffers = 170000 ; MaxDirtyBuffers = 130000 -;; uncomment next two lines if there is 4 GB system memory free -NumberOfBuffers = 340000 -MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 4 GB system memory free +;NumberOfBuffers = 340000 +;MaxDirtyBuffers = 250000 ;; Uncomment next two lines if there is 8 GB system memory free -;NumberOfBuffers = 680000 -;MaxDirtyBuffers = 500000 +NumberOfBuffers = 680000 +MaxDirtyBuffers = 500000 ;; Uncomment next two lines if there is 16 GB system memory free ;NumberOfBuffers = 1360000 ;MaxDirtyBuffers = 1000000 diff --git a/docker-compose.yml b/docker-compose.yml index 59fd8315..cf72cae0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,7 @@ services: DEFAULT_GRAPH: "http://mu.semte.ch/application" volumes: - ./data/db:/data + - ./config/virtuoso/virtuoso.ini:/data/virtuoso.ini # Note: on production override this setting restart: always logging: *default-logging labels: