From 2eff037f6bc623b8826ce8b7fea63bc21d4dc45f Mon Sep 17 00:00:00 2001 From: Paul <paul@zom.bi> Date: Tue, 12 Jan 2021 08:49:20 +0100 Subject: [PATCH] Initial commit --- 000-default.conf | 14 +++ Dockerfile | 87 ++++++++++++++ composer.local.json | 10 ++ conf/LocalSettings.local.php | 3 + conf/LocalSettings.php | 185 +++++++++++++++++++++++++++++ crontab | 12 ++ scripts/crontab-foreground | 4 + scripts/db-setup.sh | 39 ++++++ scripts/ensure-permissions.sh | 9 ++ scripts/mwjobrunner | 23 ++++ scripts/rebuildData.sh | 8 ++ scripts/removeDuplicateEntities.sh | 9 ++ scripts/run-update.sh | 4 + scripts/setupStore.sh | 8 ++ scripts/updateEntityCollation.sh | 8 ++ scripts/updateSpecialPages.sh | 8 ++ 16 files changed, 431 insertions(+) create mode 100644 000-default.conf create mode 100644 Dockerfile create mode 100644 composer.local.json create mode 100644 conf/LocalSettings.local.php create mode 100644 conf/LocalSettings.php create mode 100644 crontab create mode 100644 scripts/crontab-foreground create mode 100644 scripts/db-setup.sh create mode 100644 scripts/ensure-permissions.sh create mode 100644 scripts/mwjobrunner create mode 100644 scripts/rebuildData.sh create mode 100644 scripts/removeDuplicateEntities.sh create mode 100644 scripts/run-update.sh create mode 100644 scripts/setupStore.sh create mode 100644 scripts/updateEntityCollation.sh create mode 100644 scripts/updateSpecialPages.sh diff --git a/000-default.conf b/000-default.conf new file mode 100644 index 0000000..9853c50 --- /dev/null +++ b/000-default.conf @@ -0,0 +1,14 @@ +<VirtualHost *:80> + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + # Available loglevels: trace8, ..., trace1, debug, info, notice, warn, + # error, crit, alert, emerg. + LogLevel warn + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + Alias /wiki /var/www/html/index.php + Alias /uploads /var/www/localstore/images +</VirtualHost> diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ec9e2dd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,87 @@ +FROM mediawiki:1.35 + +ARG COMPOSER_VERSION=2.0.8 + +ENV \ + WG_SITENAME="Test Wiki" \ + WG_SCRIPT_PATH="" \ + WG_SERVER="https://wiki.example.com" \ + SEMANTIC_URL="wiki.example.com" \ + WG_ENABLE_UPLOADS="false" \ + WG_ENABLE_EMAIL="false" \ + WG_UPLOAD_PATH="/uploads" \ + WG_META_NAMESPACE="Meta" \ + WG_LANGUAGE_CODE="en" \ + MEDIAWIKI_ADMIN_USER="admin" \ + MEDIAWIKI_ADMIN_PASS="password" \ + WG_DB_TYPE="sqlite" \ + WG_DB_SERVER="" \ + WG_DB_NAME="my_wiki" \ + WG_DB_PASSWORD="password" \ + WG_DB_PREFIX="" \ + WG_DB_MWSCHEMA="" \ + WG_DATABASE_DIR="/var/www/data" \ + WG_SECRET_KEY="0000000000000000000000000000000000000000000000000000000000000000" \ + WG_EMERGENCY_CONTACT="admin@example.com" \ + WG_PASSWORD_SENDER="wiki@example.com" \ + ALLOW_PUBLIC_REGISTRATION="false" \ + ALLOW_PUBLIC_EDIT="false" \ + ALLOW_PUBLIC_READ="true" \ + DISABLE_ICONS="false" \ + DEBUG="false" + +# System dependencies for extensions +RUN set -eu; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + zip \ + unzip \ + libpq-dev \ + cron \ + rclone \ + ; \ + rm -rf /var/lib/apt/lists/* + +# PHP extensions +RUN set -eu; \ + docker-php-ext-install -j$(nproc) pgsql; \ + docker-php-ext-install -j$(nproc) pdo_pgsql + +RUN set -eu; \ + mkdir /var/www/conf; \ + mkdir -p /var/www/localstore/smwconfig; \ + mkdir -p /var/www/localstore/images + +# Non-composer based extensions +# JsonConfig required by Graph. +RUN set -euo pipefail; \ + cd /var/www/html/extensions; \ + curl https://extdist.wmflabs.org/dist/extensions/JsonConfig-REL1_35-a0bdbfb.tar.gz |tar -xz; \ + curl https://extdist.wmflabs.org/dist/extensions/Graph-REL1_35-fa2b9af.tar.gz |tar -xz; \ + curl https://extdist.wmflabs.org/dist/extensions/SubPageList3-REL1_35-622c298.tar.gz| tar -xz; \ + curl https://extdist.wmflabs.org/dist/extensions/MsUpload-REL1_35-5998b96.tar.gz| tar -xz; \ + curl https://extdist.wmflabs.org/dist/extensions/TemplateStyles-REL1_35-7743810.tar.gz| tar -xz + +# Install composer packages +RUN set -eu; \ + curl -o /tmp/composer-setup.php https://getcomposer.org/installer; \ + curl -o /tmp/composer-setup.sig https://composer.github.io/installer.sig; \ + php -r "if (hash('SHA384', file_get_contents('/tmp/composer-setup.php')) !== trim(file_get_contents('/tmp/composer-setup.sig'))) { unlink('/tmp/composer-setup.php'); echo 'Invalid installer' . PHP_EOL; exit(1); }"; \ + php /tmp/composer-setup.php --no-ansi --install-dir=/usr/local/bin --filename=composer --version=${COMPOSER_VERSION}; \ + rm -rf /tmp/composer-setup.php; \ + ln -s /var/www/conf/LocalSettings.local.php /var/www/html/LocalSettings.local.php; \ + ln -s /var/www/conf/LocalSettings.php /var/www/html/LocalSettings.php +COPY composer.local.json /var/www/html +RUN composer update --no-dev + +# Place config files +COPY conf/* /var/www/conf/ +COPY 000-default.conf /etc/apache2/sites-available + +# Place our maintenence and setup scripts +COPY scripts/* /usr/local/bin/ +RUN chmod 755 /usr/local/bin/* + +# Add crontab file in the cron directory +ADD crontab /etc/crontab +RUN chmod 0644 /etc/crontab diff --git a/composer.local.json b/composer.local.json new file mode 100644 index 0000000..7ec8c49 --- /dev/null +++ b/composer.local.json @@ -0,0 +1,10 @@ +{ + "require": { + "mediawiki/semantic-bundle": "~4.0", + "mediawiki/semantic-scribunto": "~2.1", + + "mediawiki/mermaid": "~3.0", + + "wikimedia/css-sanitizer": "2.0.1" + } +} \ No newline at end of file diff --git a/conf/LocalSettings.local.php b/conf/LocalSettings.local.php new file mode 100644 index 0000000..eca9a28 --- /dev/null +++ b/conf/LocalSettings.local.php @@ -0,0 +1,3 @@ +<?php +# This file can be replaced by mounting in a custom configuration, that will +# augment the generated config. diff --git a/conf/LocalSettings.php b/conf/LocalSettings.php new file mode 100644 index 0000000..1051cca --- /dev/null +++ b/conf/LocalSettings.php @@ -0,0 +1,185 @@ +<?php +# Further documentation for configuration settings may be found at: +# https://www.mediawiki.org/wiki/Manual:Configuration_settings + +# Protect against web entry +if ( !defined( 'MEDIAWIKI' ) ) { + exit; +} + +## Uncomment this to disable output compression +# $wgDisableOutputCompression = true; + +$wgSitename = getenv("WG_SITE_NAME"); +$wgMetaNamespace = "Meta"; + +$wgScriptPath = getenv("WG_SCRIPT_PATH"); + +$wgServer = getenv("WG_SERVER"); + +$wgUploadPath = getenv("MEDIAWIKI_UPLOAD_PATH"); + +# Article path +$wgArticlePath = "/wiki/$1"; +$wgUploadDirectory = "/var/www/localstore/images"; + +## The URL path to static resources (images, scripts, etc.) +$wgResourceBasePath = $wgScriptPath; + +## UPO means: this is also a user preference option + +$wgEnableEmail = getenv("WG_ENABLE_EMAIL"); +$wgEnableUserEmail = true; # UPO + +$wgEmergencyContact = getenv("WG_EMERGENCY_CONTACT"); +$wgPasswordSender = getenv("WG_PASSWORD_SENDER"); + +$wgEnotifUserTalk = false; # UPO +$wgEnotifWatchlist = false; # UPO +$wgEmailAuthentication = true; + +## Database settings +$wgDBtype = getenv("WG_DB_TYPE"); +$wgDBserver = getenv("WG_DB_SERVER"); +$wgDBname = getenv("WG_DB_NAME"); +$wgDBuser = getenv("WG_DB_USER"); +$wgDBpassword = getenv("WG_DB_PASSWORD"); +$wgDBport = getenv("WG_DB_PORT"); + +# Postgres specific settings +if (getenv("WG_DB_MWSCHEMA")) { + $wgDBmwschema = getenv("WG_DB_MWSCHEMA"); +} + +# SQLite-specific settings +$wgSQLiteDataDir = getenv("WG_DATABASE_DIR"); + +## Shared memory settings +if(php_sapi_name() == "cli") { + $wgMainCacheType = CACHE_NONE; +} else { + $wgMainCacheType = CACHE_ACCEL; +} +$wgMemCachedServers = []; + +$wgEnableUploads = getenv("WG_ENABLE_UPLOADS"); +$wgUseImageMagick = true; +$wgImageMagickConvertCommand = "/usr/bin/convert"; + +$wgUseInstantCommons = false; + +# telemetry +$wgPingback = false; + +## If you use ImageMagick (or any other shell command) on a +## Linux server, this will need to be set to the name of an +## available UTF-8 locale +$wgShellLocale = "C.UTF-8"; + +## Set $wgCacheDirectory to a writable directory on the web server +## to make your wiki go slightly faster. The directory should not +## be publically accessible from the web. +#$wgCacheDirectory = "$IP/cache"; + +# Site language code, should be one of the list in ./languages/data/Names.php +$wgLanguageCode = getenv("WG_LANGUAGE_CODE"); + +$wgSecretKey = getenv("WG_SECRET_KEY"); + +# Changing this will log out all existing sessions. +$wgAuthenticationTokenVersion = "1"; + +# Site upgrade key. Must be set to a string (default provided) to turn on the +# web installer while LocalSettings.php is in place +$wgUpgradeKey = "15ae97f94f551121"; + +## For attaching licensing metadata to pages, and displaying an +## appropriate copyright notice / icon. GNU Free Documentation +## License and Creative Commons licenses are supported so far. +$wgRightsPage = ""; # Set to the title of a wiki page that describes your license/copyright +$wgRightsUrl = ""; +$wgRightsText = ""; +$wgRightsIcon = ""; + +# Path to the GNU diff3 utility. Used for conflict resolution. +$wgDiff3 = "/usr/bin/diff3"; + +$wgGroupPermissions['*']['createaccount'] = !!getenv("ALLOW_PUBLIC_REGISTRATION"); +$wgGroupPermissions['*']['edit'] = !!getenv("ALLOW_PUBLIC_EDIT"); +$wgGroupPermissions['*']['read'] = !!getenv("ALLOW_PUBLIC_READ"); + +## Default skin: you can change the default skin. Use the internal symbolic +## names, ie 'vector', 'monobook': +$wgDefaultSkin = "vector"; + +# Enabled skins. +# The following skins were automatically enabled: +wfLoadSkin( 'MonoBook' ); +wfLoadSkin( 'Timeless' ); +wfLoadSkin( 'Vector' ); + +# Enabled extensions. Most of the extensions are enabled by adding +# wfLoadExtensions('ExtensionName'); +# to LocalSettings.php. Check specific extension documentation for more details. +# The following extensions were automatically enabled: +wfLoadExtension( 'CategoryTree' ); +wfLoadExtension( 'Cite' ); +wfLoadExtension( 'CodeEditor' ); +wfLoadExtension( 'ImageMap' ); +wfLoadExtension( 'InputBox' ); +wfLoadExtension( 'OATHAuth' ); +wfLoadExtension( 'ParserFunctions' ); +wfLoadExtension( 'PdfHandler' ); +wfLoadExtension( 'SyntaxHighlight_GeSHi' ); +wfLoadExtension( 'WikiEditor' ); + +# End of automatically generated settings. +# Add more configuration options below. + +# Load our extensions +wfLoadExtension( 'JsonConfig' ); # Configuration via Special Wiki Pages containing JSON +wfLoadExtension( 'TemplateStyles' ); # Embedd Styles from Wiki Pages containing CSS +wfLoadExtension( 'Scribunto' ); # Lua scripting +wfLoadExtension( 'SemanticScribunto' ); # Lua scripting using semantic information +wfLoadExtension( 'Mermaid' ); # Diagrams and flowcharts + +# scribunto configuration +$wgScribuntoDefaultEngine = 'luastandalone'; + +require_once '/var/www/html/extensions/SemanticBundle/SemanticBundle.php'; + +# Turn on SemanticMediaWiki +enableSemantics( getenv("SEMANTIC_URL") ); + +# Set Subpages on +$wgNamespacesWithSubpages[NS_MAIN] = 1; + +# Enable long error messages +if (getenv("DEBUG")) { + $wgShowExceptionDetails = true; + $wgShowDBErrorBacktrace = true; + $wgShowSQLErrors = true; +} + +# Local configuration for MediaWiki +ini_set( 'max_execution_time', 1000 ); +ini_set('memory_limit', '-1'); + +# Move the SMW config directory +$smwgConfigFileDir = '/var/www/localstore/smwconfig'; + +# Icons +$wgLogo = "favicon-135x135.png"; +$wgLogoHD = [ + "1.5x" => "favicon-202x202.png", + "2x" => "favicon-202x202.png" +]; + +# Footer icons + +if (getenv("DISABLE_ICONS")) { + unset( $wgFooterIcons['poweredby'] ); +} + +# Responsive +$wgVectorResponsive = true; \ No newline at end of file diff --git a/crontab b/crontab new file mode 100644 index 0000000..4a4835e --- /dev/null +++ b/crontab @@ -0,0 +1,12 @@ +# Reference: https://www.semantic-mediawiki.org/wiki/Cron_jobs + +# Every day at 07:45, Update Special Pages +45 7 * * * root /usr/local/bin/updateSpecialPages.sh > /proc/1/fd/1 2>/proc/1/fd/2 + +# SemanticMediaWiki recommended maintenance +30 * * * * root /usr/local/bin/updateEntityCollation.sh > /proc/1/fd/1 2>/proc/1/fd/2 +15 9 * * SAT root /usr/local/bin/rebuildData.sh > /proc/1/fd/1 2>/proc/1/fd/2 +15 9 * * SUN root /usr/local/bin/setupStore.sh > /proc/1/fd/1 2>/proc/1/fd/2 +15 3 1 * * root php /usr/local/bin/removeDuplicateEntities.sh > /proc/1/fd/1 2>/proc/1/fd/2 + +# file requires a trailing newline! diff --git a/scripts/crontab-foreground b/scripts/crontab-foreground new file mode 100644 index 0000000..a2ae5ea --- /dev/null +++ b/scripts/crontab-foreground @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +cron -f -L 15 \ No newline at end of file diff --git a/scripts/db-setup.sh b/scripts/db-setup.sh new file mode 100644 index 0000000..122fcab --- /dev/null +++ b/scripts/db-setup.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +cd /var/www/html/ + +# The install script doesn't want there to be a LocalSettings.php file +echo Killing LocalSettings.php +echo + +rm /var/www/html/LocalSettings.php + +echo Install.php +echo + +php maintenance/install.php --dbtype "${MEDIAWIKI_DB_TYPE}" --dbname "${MEDIAWIKI_DB_NAME}" --dbuser "${MEDIAWIKI_DB_USER}" --dbport "${MEDIAWIKI_DB_PORT}" --dbpass "${MEDIAWIKI_DB_PASSWORD}" --scriptpath "/var/www/html/" --dbserver "${MEDIAWIKI_DB_HOST}" --pass "${MEDIAWIKI_ADMIN_PASS}" --dbpath "${MEDIAWIKI_DATABASE_DIR}" "${MEDIAWIKI_SITE_NAME}" "${MEDIAWIKI_ADMIN_USER}" + +# Now, we're going to replace the LocalSettings.php file that install.php just generated with ours +# This way, update.php will work. +rm /var/www/html/LocalSettings.php +ln -s /var/www/conf/LocalSettings.php /var/www/html/LocalSettings.php + +echo LocalSettings.php restored +echo + +echo Creating localstore +echo + +# Copy the htaccess file +mkdir -p /var/www/localstore/images +mkdir -p /var/www/localstore/smwconfig +cp /var/www/html/images/* /var/www/localstore/images +chown -R www-data:www-data /var/www/localstore + +# Run update.php, to set up all of the extensions + +echo update.php +echo + +php maintenance/update.php --quick \ No newline at end of file diff --git a/scripts/ensure-permissions.sh b/scripts/ensure-permissions.sh new file mode 100644 index 0000000..9700094 --- /dev/null +++ b/scripts/ensure-permissions.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +# Copy the htaccess file +mkdir -p /var/www/localstore/images +mkdir -p /var/www/localstore/smwconfig +cp /var/www/html/images/* /var/www/localstore/images +chown -R www-data:www-data /var/www/localstore +chown -R www-data:www-data /var/www/data diff --git a/scripts/mwjobrunner b/scripts/mwjobrunner new file mode 100644 index 0000000..f2871f1 --- /dev/null +++ b/scripts/mwjobrunner @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +# From https://www.mediawiki.org/wiki/Manual:Job_queue +IP=/var/www/html +RJ=$IP/maintenance/runJobs.php +echo Starting job service... +# Wait a minute after the server starts up to give other processes time to get started +sleep 60 +echo Started. +while true; do + # Job types that need to be run ASAP mo matter how many of them are in the queue + # Those jobs should be very "cheap" to run + php $RJ --type="enotifNotify" + php $RJ --type="htmlCacheUpdate" --maxjobs=50 + # Everything else, limit the number of jobs on each batch + # The --wait parameter will pause the execution here until new jobs are added, + # to avoid running the loop without anything to do + php $RJ --wait --maxjobs=10 + # Wait some seconds to let the CPU do other things, like handling web requests, etc + echo Waiting for 10 seconds... + sleep 10 +done \ No newline at end of file diff --git a/scripts/rebuildData.sh b/scripts/rebuildData.sh new file mode 100644 index 0000000..c6bd418 --- /dev/null +++ b/scripts/rebuildData.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +echo rebuildData +echo + +php /var/www/html/extensions/SemanticMediaWiki/maintenance/rebuildData.php -d 100 +echo rebuildData finished \ No newline at end of file diff --git a/scripts/removeDuplicateEntities.sh b/scripts/removeDuplicateEntities.sh new file mode 100644 index 0000000..c5a5695 --- /dev/null +++ b/scripts/removeDuplicateEntities.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +echo removeDuplicateEntities +echo + +/usr/local/bin/php /var/www/html/extensions/SemanticMediaWiki/maintenance/removeDuplicateEntities.php +echo removeDuplicateEntities finished + diff --git a/scripts/run-update.sh b/scripts/run-update.sh new file mode 100644 index 0000000..6326a2b --- /dev/null +++ b/scripts/run-update.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +/usr/local/bin/php maintenance/update.php \ No newline at end of file diff --git a/scripts/setupStore.sh b/scripts/setupStore.sh new file mode 100644 index 0000000..539377a --- /dev/null +++ b/scripts/setupStore.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +echo setupStore +echo + +/usr/local/bin/php /var/www/html/extensions/SemanticMediaWiki/maintenance/setupStore.php --skip-import +echo setupStore finished \ No newline at end of file diff --git a/scripts/updateEntityCollation.sh b/scripts/updateEntityCollation.sh new file mode 100644 index 0000000..28288ff --- /dev/null +++ b/scripts/updateEntityCollation.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +echo updateEntityCollation +echo + +/usr/local/bin/php /var/www/html/extensions/SemanticMediaWiki/maintenance/updateEntityCollation.php +echo updateEntityCollation finished \ No newline at end of file diff --git a/scripts/updateSpecialPages.sh b/scripts/updateSpecialPages.sh new file mode 100644 index 0000000..9ac6ec6 --- /dev/null +++ b/scripts/updateSpecialPages.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +echo updateSpecialPages +echo + +/usr/local/bin/php /var/www/html/maintenance/updateSpecialPages.php +echo updateSpecialPages finished \ No newline at end of file