diff options
Diffstat (limited to 'MLEB/Translate/utils')
34 files changed, 1539 insertions, 558 deletions
diff --git a/MLEB/Translate/utils/ArrayFlattener.php b/MLEB/Translate/utils/ArrayFlattener.php index c5e61769..db57b4f5 100644 --- a/MLEB/Translate/utils/ArrayFlattener.php +++ b/MLEB/Translate/utils/ArrayFlattener.php @@ -78,8 +78,8 @@ class ArrayFlattener { * @return bool|string */ public function flattenCLDRPlurals( $messages ) { - $pluralKeys = false; - $nonPluralKeys = false; + $hasNonPluralKeys = false; + $pluralKeys = []; foreach ( $messages as $key => $value ) { if ( is_array( $value ) ) { // Plurals can only happen in the lowest level of the structure @@ -88,9 +88,9 @@ class ArrayFlattener { // Check if we find any reserved plural keyword if ( isset( self::$pluralWords[$key] ) ) { - $pluralKeys = true; + $pluralKeys[] = $key; } else { - $nonPluralKeys = true; + $hasNonPluralKeys = true; } } @@ -100,7 +100,12 @@ class ArrayFlattener { } // Mixed plural keys with other keys, should not happen - if ( $nonPluralKeys ) { + if ( $hasNonPluralKeys ) { + // Allow `other` with other keys, as long it is is only one of the reserved ones + if ( $pluralKeys === [ 'other' ] ) { + return false; + } + $keys = implode( ', ', array_keys( $messages ) ); throw new MWException( "Reserved plural keywords mixed with other keys: $keys." ); } @@ -137,7 +142,7 @@ class ArrayFlattener { if ( !is_array( $value ) ) { $plurals = $this->unflattenCLDRPlurals( $key, $value ); } - if ( $plurals ) { + if ( is_array( $plurals ) ) { $unflattenedPlurals += $plurals; } else { $unflattenedPlurals[$key] = $value; @@ -179,7 +184,7 @@ class ArrayFlattener { } /** - * Converts the MediaWiki plural syntax to array of CLDR style plurals + * Converts the plural syntax to array of CLDR style plurals * * @param string $key Message key prefix * @param string $message The plural string @@ -196,7 +201,7 @@ class ArrayFlattener { * Replace all variables with placeholders. Possible source of bugs * if other characters that given below are used. */ - $regex = '~\{[a-zA-Z_-]+}~'; + $regex = '/\{[a-z_-]+}/i'; $placeholders = []; $match = []; @@ -232,15 +237,11 @@ class ArrayFlattener { * multiple plural bocks which don't have the same set of keys. */ $pluralChoice = implode( '|', array_keys( self::$pluralWords ) ); - $regex = "~($pluralChoice)\s*=\s*(.+)~s"; + $regex = "~($pluralChoice)\s*=\s*(.*)~s"; foreach ( $matches as $ph => $plu ) { $forms = explode( '|', $plu[1] ); foreach ( $forms as $form ) { - if ( $form === '' ) { - continue; - } - $match = []; if ( preg_match( $regex, $form, $match ) ) { $formWord = "$key{$this->sep}{$match[1]}"; @@ -265,7 +266,8 @@ class ArrayFlattener { } if ( !isset( $alts["$key{$this->sep}other"] ) ) { - wfWarn( "Other not set for key $key" ); + // Ensure other form is always present, even if missing from the translation + $alts["$key{$this->sep}other"] = end( $alts ); } return $alts; diff --git a/MLEB/Translate/utils/ExternalMessageSourceStateComparator.php b/MLEB/Translate/utils/ExternalMessageSourceStateComparator.php index 5b24b5e0..a6b53f4f 100644 --- a/MLEB/Translate/utils/ExternalMessageSourceStateComparator.php +++ b/MLEB/Translate/utils/ExternalMessageSourceStateComparator.php @@ -7,33 +7,51 @@ * @license GPL-2.0-or-later * @since 2013.12 */ + +use MediaWiki\Extensions\Translate\MessageSync\MessageSourceChange; +use MediaWiki\Extensions\Translate\Utilities\StringComparators\StringComparator; + class ExternalMessageSourceStateComparator { /** Process all languages supported by the message group */ - const ALL_LANGUAGES = 'all languages'; + public const ALL_LANGUAGES = 'all languages'; - protected $changes = []; + /** + * @var StringComparator + */ + protected $stringComparator; /** - * Finds changes in external sources compared to wiki state. + * @param StringComparator $stringComparator + */ + public function __construct( StringComparator $stringComparator ) { + $this->stringComparator = $stringComparator; + } + + /** + * Finds modifications in external sources compared to wiki state. * - * The returned array is as following: - * - First level is indexed by language code - * - Second level is indexed by change type: - * - - addition (new message in the file) - * - - deletion (message in wiki not present in the file) - * - - change (difference in content) - * - Third level is a list of changes - * - Fourth level is change properties - * - - key (the message key) - * - - content (the message content in external source, null for deletions) + * The MessageSourceChange object returned stores the following about each modification, + * - First level of classification is the language code + * - Second level of classification is the type of modification, + * - addition (new message in the file) + * - deletion (message in wiki not present in the file) + * - change (difference in content) + * - rename (message key is modified) + * - Third level is a list of modifications + * - For each modification, the following is saved, + * - key (the message key) + * - content (the message content in external source, null for deletions) + * - matched_to (present in case of renames, key of the matched message) + * - similarity (present in case of renames, similarity % with the matched message) + * - previous_state (present in case of renames, state of the message before rename) * * @param FileBasedMessageGroup $group * @param array|string $languages - * @throws MWException - * @return array array[language code][change type] = change. + * @throws InvalidArgumentException + * @return MessageSourceChange */ public function processGroup( FileBasedMessageGroup $group, $languages ) { - $this->changes = []; + $changes = new MessageSourceChange(); $processAll = false; if ( $languages === self::ALL_LANGUAGES ) { @@ -47,7 +65,7 @@ class ExternalMessageSourceStateComparator { $languages = array_keys( $languages ); } elseif ( !is_array( $languages ) ) { - throw new MWException( 'Invalid input given for $languages' ); + throw new InvalidArgumentException( 'Invalid input given for $languages' ); } // Process the source language before others. Source language might not @@ -58,23 +76,25 @@ class ExternalMessageSourceStateComparator { $index = array_search( $sourceLanguage, $languages ); if ( $processAll || $index !== false ) { unset( $languages[$index] ); - $this->processLanguage( $group, $sourceLanguage ); + $this->processLanguage( $group, $sourceLanguage, $changes ); } - foreach ( $languages as $code ) { - $this->processLanguage( $group, $code ); + foreach ( $languages as $language ) { + $this->processLanguage( $group, $language, $changes ); } - return $this->changes; + return $changes; } - protected function processLanguage( FileBasedMessageGroup $group, $code ) { - $cache = new MessageGroupCache( $group, $code ); + protected function processLanguage( + FileBasedMessageGroup $group, $language, MessageSourceChange $changes + ) { + $cache = $group->getMessageGroupCache( $language ); $reason = 0; if ( !$cache->isValid( $reason ) ) { - $this->addMessageUpdateChanges( $group, $code, $reason, $cache ); + $this->addMessageUpdateChanges( $group, $language, $changes, $reason, $cache ); - if ( !isset( $this->changes[$code] ) ) { + if ( $changes->getModificationsForLanguage( $language ) === [] ) { /* Update the cache immediately if file and wiki state match. * Otherwise the cache will get outdated compared to file state * and will give false positive conflicts later. */ @@ -93,38 +113,40 @@ class ExternalMessageSourceStateComparator { * Now we must try to guess what in earth has driven the file state and * wiki state out of sync. Then we must compile list of events that would * bring those to sync. Types of events are addition, deletion, (content) - * change and possible rename in the future. After that the list of events - * are stored for later processing of a translation administrator, who can - * decide what actions to take on those events to bring the state more or - * less in sync. + * change and key renames. After that the list of events are stored for + * later processing of a translation administrator, who can decide what + * actions to take on those events to bring the state more or less in sync. * * @param FileBasedMessageGroup $group - * @param string $code Language code. + * @param string $language + * @param MessageSourceChange $changes * @param int $reason * @param MessageGroupCache $cache - * @throws MWException + * @throws RuntimeException */ - protected function addMessageUpdateChanges( FileBasedMessageGroup $group, $code, - $reason, $cache + protected function addMessageUpdateChanges( + FileBasedMessageGroup $group, $language, MessageSourceChange $changes, $reason, $cache ) { /* This throws a warning if message definitions are not yet * cached and will read the file for definitions. */ - MediaWiki\suppressWarnings(); - $wiki = $group->initCollection( $code ); - MediaWiki\restoreWarnings(); + Wikimedia\suppressWarnings(); + $wiki = $group->initCollection( $language ); + Wikimedia\restoreWarnings(); $wiki->filter( 'hastranslation', false ); $wiki->loadTranslations(); $wikiKeys = $wiki->getMessageKeys(); + $sourceLanguage = $group->getSourceLanguage(); // By-pass cached message definitions /** @var FFS $ffs */ $ffs = $group->getFFS(); - if ( $code === $group->getSourceLanguage() && !$ffs->exists( $code ) ) { - $path = $group->getSourceFilePath( $code ); - throw new MWException( "Source message file for {$group->getId()} does not exist: $path" ); + '@phan-var SimpleFFS $ffs'; + if ( $language === $sourceLanguage && !$ffs->exists( $language ) ) { + $path = $group->getSourceFilePath( $language ); + throw new RuntimeException( "Source message file for {$group->getId()} does not exist: $path" ); } - $file = $ffs->read( $code ); + $file = $ffs->read( $language ); // Does not exist if ( $file === false ) { @@ -136,7 +158,7 @@ class ExternalMessageSourceStateComparator { $id = $group->getId(); $ffsClass = get_class( $ffs ); - error_log( "$id has an FFS ($ffsClass) - it didn't return cake for $code" ); + error_log( "$id has an FFS ($ffsClass) - it didn't return cake for $language" ); return; } @@ -146,6 +168,7 @@ class ExternalMessageSourceStateComparator { $common = array_intersect( $fileKeys, $wikiKeys ); $supportsFuzzy = $ffs->supportsFuzzy(); + $changesToRemove = []; foreach ( $common as $key ) { $sourceContent = $file['MESSAGES'][$key]; @@ -185,16 +208,36 @@ class ExternalMessageSourceStateComparator { } } - $this->addChange( 'change', $code, $key, $sourceContent ); + if ( $language !== $sourceLanguage ) { + // Assuming that this is the old key, lets check if it has a corresponding + // rename in the source language. The key of the matching message will be + // the new renamed key. + $renameMsg = $changes->getMatchedMessage( $sourceLanguage, $key ); + if ( $renameMsg !== null ) { + // Rename present in source language but this message has a content change + // with the OLD key in a non-source language. We will not process this + // here but add it as a rename instead. This way, the key will be renamed + // and then the content updated. + $this->addNonSourceRenames( + $changes, $key, $renameMsg['key'], $sourceContent, $wikiContent, $language + ); + $changesToRemove[] = $key; + continue; + } + } + $changes->addChange( $language, $key, $sourceContent ); } + $changes->removeChanges( $language, $changesToRemove ); + $added = array_diff( $fileKeys, $wikiKeys ); foreach ( $added as $key ) { $sourceContent = $file['MESSAGES'][$key]; if ( trim( $sourceContent ) === '' ) { continue; } - $this->addChange( 'addition', $code, $key, $sourceContent ); + + $changes->addAddition( $language, $key, $sourceContent ); } /* Should the cache not exist, don't consider the messages @@ -209,15 +252,195 @@ class ExternalMessageSourceStateComparator { * must be a newly made in the wiki. */ continue; } - $this->addChange( 'deletion', $code, $key, null ); + $changes->addDeletion( $language, $key, $wiki[$key]->translation() ); } } + + if ( $language === $sourceLanguage ) { + $this->findAndMarkSourceRenames( $changes, $language ); + } else { + // Non source language + $this->checkNonSourceAdditionsForRename( + $changes, $sourceLanguage, $language, $wiki, $wikiKeys + ); + } } - protected function addChange( $type, $language, $key, $content ) { - $this->changes[$language][$type][] = [ + /** + * For non source languages, we look at additions and see if they have been + * added as renames in the source language. + * @param MessageSourceChange $changes + * @param string $sourceLanguage + * @param string $targetLanguage + * @param MessageCollection $wiki + * @param string[] $wikiKeys + */ + private function checkNonSourceAdditionsForRename( + MessageSourceChange $changes, $sourceLanguage, $targetLanguage, MessageCollection $wiki, $wikiKeys + ) { + $additions = $changes->getAdditions( $targetLanguage ); + if ( $additions === [] ) { + return; + } + + $additionsToRemove = []; + $deletionsToRemove = []; + foreach ( $additions as $addedMsg ) { + $addedMsgKey = $addedMsg['key']; + + // Check if this key is renamed in source. + $renamedSourceMsg = $changes->findMessage( + $sourceLanguage, $addedMsgKey, [ MessageSourceChange::RENAME ] + ); + + if ( $renamedSourceMsg === null ) { + continue; + } + + // Since this key is new, and is present in the renames for the source language, + // we will add it as a rename. + $deletedSource = $changes->getMatchedMessage( $sourceLanguage, $renamedSourceMsg['key'] ); + $deletedMsgKey = $deletedSource['key']; + $deletedMsg = $changes->findMessage( + $targetLanguage, $deletedMsgKey, [ MessageSourceChange::DELETION ] + ); + + // Sometimes when the cache does not have the translations, the deleted message + // is not added in the translations. It is also possible that for this non-source + // language the key has not been removed. + if ( $deletedMsg === null ) { + $content = ''; + if ( array_search( $deletedMsgKey, $wikiKeys ) !== false ) { + $content = $wiki[ $deletedMsgKey ]->translation(); + } + $deletedMsg = [ + 'key' => $deletedMsgKey, + 'content' => $content + ]; + } + + $similarityPercent = $this->stringComparator->getSimilarity( + $addedMsg['content'], $deletedMsg['content'] + ); + + $changes->addRename( $targetLanguage, [ + 'key' => $addedMsgKey, + 'content' => $addedMsg['content'] + ], [ + 'key' => $deletedMsgKey, + 'content' => $deletedMsg['content'] + ], $similarityPercent ); + + $deletionsToRemove[] = $deletedMsgKey; + $additionsToRemove[] = $addedMsgKey; + } + + $changes->removeAdditions( $targetLanguage, $additionsToRemove ); + $changes->removeDeletions( $targetLanguage, $deletionsToRemove ); + } + + /** + * Check for renames and add them to the changes. To identify renames we need to + * compare the contents of the added messages with the deleted ones and identify + * messages that match. + * @param MessageSourcechange $changes + * @param string $sourceLanguage + */ + private function findAndMarkSourceRenames( MessageSourceChange $changes, $sourceLanguage ) { + // Now check for renames. To identify renames we need to compare + // the contents of the added messages with the deleted ones and + // identify messages that match. + $deletions = $changes->getDeletions( $sourceLanguage ); + $additions = $changes->getAdditions( $sourceLanguage ); + if ( $deletions === [] || $additions === [] ) { + return; + } + + // This array contains a dictionary with matching renames in the following structure - + // [ A1|D1 => 1.0, A1|D2 => 0.95, A2|D1 => 0.95 ] + $potentialRenames = []; + foreach ( $additions as $addedMsg ) { + $addedMsgKey = $addedMsg['key']; + + foreach ( $deletions as $deletedMsg ) { + $similarityPercent = $this->stringComparator->getSimilarity( + $addedMsg['content'], $deletedMsg['content'] + ); + + if ( $changes->areStringsSimilar( $similarityPercent ) ) { + $potentialRenames[ $addedMsgKey . '|' . $deletedMsg['key'] ] = $similarityPercent; + } + } + } + + $this->matchRenames( $changes, $potentialRenames, $sourceLanguage ); + } + + /** + * Adds non source language renames to the list of changes + * @param MessageSourceChange $changes + * @param string $key + * @param string $renameKey + * @param string $sourceContent + * @param string $wikiContent + * @param string $language + */ + private function addNonSourceRenames( + MessageSourceChange $changes, $key, $renameKey, $sourceContent, $wikiContent, $language + ) { + $addedMsg = [ + 'key' => $renameKey, + 'content' => $sourceContent + ]; + + $removedMsg = [ 'key' => $key, - 'content' => $content, + 'content' => $wikiContent ]; + + $similarityPercent = $this->stringComparator->getSimilarity( + $sourceContent, $wikiContent + ); + $changes->addRename( $language, $addedMsg, $removedMsg, $similarityPercent ); + } + + /** + * Identifies which added message to be associated with the deleted message based on + * similarity percentage. + * + * We sort the $trackRename array on the similarity percentage and then start adding the + * messages as renames. + * @param MessageSourceChange $changes + * @param array $trackRename + * @param string $language + */ + private function matchRenames( MessageSourceChange $changes, array $trackRename, $language ) { + arsort( $trackRename, SORT_NUMERIC ); + + $alreadyRenamed = $additionsToRemove = $deletionsToRemove = []; + foreach ( $trackRename as $key => $similarityPercent ) { + list( $addKey, $deleteKey ) = explode( '|', $key, 2 ); + if ( isset( $alreadyRenamed[ $addKey ] ) || isset( $alreadyRenamed[ $deleteKey ] ) ) { + // Already mapped with another name. + continue; + } + + // Using key should be faster than saving values and searching for them in the array. + $alreadyRenamed[ $addKey ] = 1; + $alreadyRenamed[ $deleteKey ] = 1; + + $addMsg = $changes->findMessage( $language, $addKey, [ MessageSourceChange::ADDITION ] ); + $deleteMsg = $changes->findMessage( $language, $deleteKey, [ MessageSourceChange::DELETION ] ); + + $changes->addRename( $language, $addMsg, $deleteMsg, $similarityPercent ); + + // @phan-suppress-next-line PhanTypeArraySuspiciousNullable + $additionsToRemove[] = $addMsg['key']; + // @phan-suppress-next-line PhanTypeArraySuspiciousNullable + $deletionsToRemove[] = $deleteMsg['key']; + } + + $changes->removeAdditions( $language, $additionsToRemove ); + $changes->removeDeletions( $language, $deletionsToRemove ); } } diff --git a/MLEB/Translate/utils/ExternalMessageSourceStateImporter.php b/MLEB/Translate/utils/ExternalMessageSourceStateImporter.php index 495c3fd7..51674921 100644 --- a/MLEB/Translate/utils/ExternalMessageSourceStateImporter.php +++ b/MLEB/Translate/utils/ExternalMessageSourceStateImporter.php @@ -7,56 +7,67 @@ * @license GPL-2.0-or-later * @since 2016.02 */ + +use MediaWiki\Extensions\Translate\MessageSync\MessageSourceChange; + class ExternalMessageSourceStateImporter { - public function importSafe( $changeData ) { + /** + * @param MessageSourceChange[] $changeData + * @return array + */ + public function importSafe( array $changeData ) { $processed = []; $skipped = []; $jobs = []; $jobs[] = MessageIndexRebuildJob::newJob(); + /** + * @var MessageSourceChange $changesForGroup + */ foreach ( $changeData as $groupId => $changesForGroup ) { + /** + * @var FileBasedMessageGroup + */ $group = MessageGroups::getGroup( $groupId ); if ( !$group ) { unset( $changeData[$groupId] ); continue; } + '@phan-var FileBasedMessageGroup $group'; - $processed[$groupId] = 0; + $processed[$groupId] = []; + $languages = $changesForGroup->getLanguages(); - foreach ( $changesForGroup as $languageCode => $changesForLanguage ) { - if ( !self::isSafe( $changesForLanguage ) ) { + foreach ( $languages as $language ) { + if ( !self::isSafe( $changesForGroup, $language ) ) { + // changes other than additions were present $skipped[$groupId] = true; continue; } - if ( !isset( $changesForLanguage['addition'] ) ) { + $additions = $changesForGroup->getAdditions( $language ); + if ( $additions === [] ) { continue; } - foreach ( $changesForLanguage['addition'] as $addition ) { - $namespace = $group->getNamespace(); - $name = "{$addition['key']}/$languageCode"; - - $title = Title::makeTitleSafe( $namespace, $name ); - if ( !$title ) { - wfWarn( "Invalid title for group $groupId key {$addition['key']}" ); - continue; - } + [ $groupJobs, $groupProcessed ] = $this->createMessageUpdateJobs( + $group, $additions, $language + ); - $jobs[] = MessageUpdateJob::newJob( $title, $addition['content'] ); - $processed[$groupId]++; - } - - unset( $changeData[$groupId][$languageCode] ); + $jobs = array_merge( $jobs, $groupJobs ); + $processed[$groupId][$language] = $groupProcessed; - $cache = new MessageGroupCache( $groupId, $languageCode ); - $cache->create(); + $changesForGroup->removeChangesForLanguage( $language ); + $group->getMessageGroupCache( $language )->create(); } } // Remove groups where everything was imported - $changeData = array_filter( $changeData ); + $changeData = array_filter( $changeData, function ( MessageSourceChange $change ) { + return $change->getAllModifications() !== []; + } ); + // Remove groups with no imports $processed = array_filter( $processed ); @@ -72,13 +83,44 @@ class ExternalMessageSourceStateImporter { ]; } - protected static function isSafe( array $changesForLanguage ) { - foreach ( array_keys( $changesForLanguage ) as $changeType ) { - if ( $changeType !== 'addition' ) { - return false; + /** + * Checks if changes for a language in a group are safe. + * @param MessageSourceChange $changesForGroup + * @param string $language + * @return bool + */ + public static function isSafe( MessageSourceChange $changesForGroup, $language ) { + return $changesForGroup->hasOnly( $language, MessageSourceChange::ADDITION ); + } + + /** + * Creates MessagUpdateJobs additions for a language under a group + * + * @param MessageGroup $group + * @param string[][] $additions + * @param string $language + * @return array + */ + private function createMessageUpdateJobs( + MessageGroup $group, array $additions, string $language + ) { + $groupId = $group->getId(); + $jobs = []; + $processed = 0; + foreach ( $additions as $addition ) { + $namespace = $group->getNamespace(); + $name = "{$addition['key']}/$language"; + + $title = Title::makeTitleSafe( $namespace, $name ); + if ( !$title ) { + wfWarn( "Invalid title for group $groupId key {$addition['key']}" ); + continue; } + + $jobs[] = MessageUpdateJob::newJob( $title, $addition['content'] ); + $processed++; } - return true; + return [ $jobs, $processed ]; } } diff --git a/MLEB/Translate/utils/Font.php b/MLEB/Translate/utils/FCFontFinder.php index 37fa4ac7..37fa4ac7 100644 --- a/MLEB/Translate/utils/Font.php +++ b/MLEB/Translate/utils/FCFontFinder.php diff --git a/MLEB/Translate/utils/FuzzyBot.php b/MLEB/Translate/utils/FuzzyBot.php deleted file mode 100644 index 093e39e1..00000000 --- a/MLEB/Translate/utils/FuzzyBot.php +++ /dev/null @@ -1,25 +0,0 @@ -<?php -/** - * Do it all maintenance account - * - * @file - * @author Niklas Laxström - * @copyright Copyright © 2012-2013, Niklas Laxström - * @license GPL-2.0-or-later - */ - -/** - * FuzzyBot - the misunderstood workhorse. - * @since 2012-01-02 - */ -class FuzzyBot { - public static function getUser() { - return User::newSystemUser( self::getName(), [ 'steal' => true ] ); - } - - public static function getName() { - global $wgTranslateFuzzyBotName; - - return $wgTranslateFuzzyBotName; - } -} diff --git a/MLEB/Translate/utils/JsSelectToInput.php b/MLEB/Translate/utils/JsSelectToInput.php index 24f30bc1..83d5cc65 100644 --- a/MLEB/Translate/utils/JsSelectToInput.php +++ b/MLEB/Translate/utils/JsSelectToInput.php @@ -100,7 +100,9 @@ class JsSelectToInput { $html = Xml::element( 'input', [ 'type' => 'button', 'value' => wfMessage( $msg )->text(), - 'onclick' => Xml::encodeJsCall( 'appendFromSelect', [ $source, $target ] ) + 'class' => 'mw-translate-jssti', + 'data-translate-jssti-sourceid' => $source, + 'data-translate-jssti-targetid' => $target ] ); return $html; diff --git a/MLEB/Translate/utils/MessageChangeStorage.php b/MLEB/Translate/utils/MessageChangeStorage.php index 5c23a3a6..6fbd8a0a 100644 --- a/MLEB/Translate/utils/MessageChangeStorage.php +++ b/MLEB/Translate/utils/MessageChangeStorage.php @@ -1,6 +1,6 @@ <?php /** - * Handles storage of message change files. + * Handles storage / retrieval of data from message change files. * * @author Niklas Laxström * @license GPL-2.0-or-later @@ -8,23 +8,28 @@ * @file */ +use MediaWiki\Extensions\Translate\MessageSync\MessageSourceChange; + class MessageChangeStorage { - const DEFAULT_NAME = 'default'; + public const DEFAULT_NAME = 'default'; /** * Writes change array as a serialized file. * - * @param array $array Array of changes as returned by processGroup + * @param MessageSourceChange[] $changes Array of changes as returned by processGroup * indexed by message group id. * @param string $file Which file to use. */ - public static function writeChanges( $array, $file ) { + public static function writeChanges( array $changes, $file ) { $cache = \Cdb\Writer::open( $file ); - $keys = array_keys( $array ); - $cache->set( '#keys', serialize( $keys ) ); + $keys = array_keys( $changes ); + $cache->set( '#keys', TranslateUtils::serialize( $keys ) ); - foreach ( $array as $key => $value ) { - $value = serialize( $value ); + /** + * @var MessageSourceChange $change + */ + foreach ( $changes as $key => $change ) { + $value = TranslateUtils::serialize( $change->getAllModifications() ); $cache->set( $key, $value ); } $cache->close(); @@ -37,7 +42,7 @@ class MessageChangeStorage { * @return bool */ public static function isValidCdbName( $name ) { - return preg_match( '/^[a-zA-Z_-]{1,100}$/', $name ); + return preg_match( '/^[a-z_-]{1,100}$/i', $name ); } /** @@ -49,4 +54,101 @@ class MessageChangeStorage { public static function getCdbPath( $name ) { return TranslateUtils::cacheFile( "messagechanges.$name.cdb" ); } + + /** + * Fetches changes for a group from the message change file. + * @param string $cdbPath Path of the cdb file. + * @param string $groupId Group Id + * @return MessageSourceChange + */ + public static function getGroupChanges( $cdbPath, $groupId ) { + $reader = self::getCdbReader( $cdbPath ); + if ( $reader === null ) { + return MessageSourceChange::loadModifications( [] ); + } + + $groups = TranslateUtils::deserialize( $reader->get( '#keys' ) ); + + if ( !in_array( $groupId, $groups, true ) ) { + throw new InvalidArgumentException( "Group Id - '$groupId' not found in cdb file " . + "(path: $cdbPath)." ); + } + + return MessageSourceChange::loadModifications( + TranslateUtils::deserialize( $reader->get( $groupId ) ) + ); + } + + /** + * Writes changes for a group. Has to read the changes first from the file, + * and then re-write them to the file. + * @param MessageSourceChange $changes + * @param string $groupId Group Id + * @param string $cdbPath Path of the cdb file. + */ + public static function writeGroupChanges( MessageSourceChange $changes, $groupId, $cdbPath ) { + $reader = self::getCdbReader( $cdbPath ); + if ( $reader === null ) { + return; + } + + $groups = TranslateUtils::deserialize( $reader->get( '#keys' ) ); + + $allChanges = []; + foreach ( $groups as $id ) { + $allChanges[$id] = MessageSourceChange::loadModifications( + TranslateUtils::deserialize( $reader->get( $id ) ) + ); + } + $allChanges[$groupId] = $changes; + + self::writeChanges( $allChanges, $cdbPath ); + } + + /** + * Validate and return a reader reference to the CDB file + * @param string $cdbPath + * @return \Cdb\Reader + */ + private static function getCdbReader( $cdbPath ) { + // File not found, probably no changes. + if ( !file_exists( $cdbPath ) ) { + return null; + } + + return \Cdb\Reader::open( $cdbPath ); + } + + /** + * Gets the last modified time for the CDB file. + * + * @param string $cdbPath + * @return int time of last modification (Unix timestamp) + */ + public static function getLastModifiedTime( $cdbPath ) { + // File not found + if ( !file_exists( $cdbPath ) ) { + return null; + } + + $stat = stat( $cdbPath ); + + return $stat['mtime']; + } + + /** + * Checks if the CDB file has been modified since the time given. + * @param string $cdbPath + * @param int $time Unix timestamp + * @return bool + */ + public static function isModifiedSince( $cdbPath, $time ) { + $lastModifiedTime = self::getLastModifiedTime( $cdbPath ); + + if ( $lastModifiedTime === null ) { + throw new InvalidArgumentException( "CDB file not found - $cdbPath" ); + } + + return $lastModifiedTime <= $time; + } } diff --git a/MLEB/Translate/utils/MessageGroupCache.php b/MLEB/Translate/utils/MessageGroupCache.php index f0b9c4bd..fdb1fc01 100644 --- a/MLEB/Translate/utils/MessageGroupCache.php +++ b/MLEB/Translate/utils/MessageGroupCache.php @@ -1,9 +1,7 @@ <?php /** - * Code for caching the messages of file based message groups. * @file * @author Niklas Laxström - * @copyright Copyright © 2009-2013 Niklas Laxström * @license GPL-2.0-or-later */ @@ -11,17 +9,18 @@ * Caches messages of file based message group source file. Can also track * that the cache is up to date. Parsing the source files can be slow, so * constructing CDB cache makes accessing that data constant speed regardless - * of the actual format. + * of the actual format. This also avoid having to deal with potentially unsafe + * external files during web requests. * * @ingroup MessageGroups */ class MessageGroupCache { - const NO_SOURCE = 1; - const NO_CACHE = 2; - const CHANGED = 3; + public const NO_SOURCE = 1; + public const NO_CACHE = 2; + public const CHANGED = 3; /** - * @var MessageGroup + * @var FileBasedMessageGroup */ protected $group; @@ -36,17 +35,24 @@ class MessageGroupCache { protected $code; /** + * @var string + */ + private $cacheFilePath; + + /** * Contructs a new cache object for given group and language code. - * @param string|FileBasedMessageGroup $group Group object or id. - * @param string $code Language code. Default value 'en'. + * @param FileBasedMessageGroup $group + * @param string $code Language code. + * @param string $cacheFilePath */ - public function __construct( $group, $code = 'en' ) { - if ( is_object( $group ) ) { - $this->group = $group; - } else { - $this->group = MessageGroups::getGroup( $group ); - } + public function __construct( + FileBasedMessageGroup $group, + string $code, + string $cacheFilePath + ) { + $this->group = $group; $this->code = $code; + $this->cacheFilePath = $cacheFilePath; } /** @@ -54,22 +60,7 @@ class MessageGroupCache { * @return bool */ public function exists() { - $old = $this->getOldCacheFileName(); - $new = $this->getCacheFileName(); - $exists = file_exists( $new ); - - if ( $exists ) { - return true; - } - - // Perform migration if possible - if ( file_exists( $old ) ) { - wfMkdirParents( dirname( $new ) ); - rename( $old, $new ); - return true; - } - - return false; + return file_exists( $this->getCacheFilePath() ); } /** @@ -77,10 +68,19 @@ class MessageGroupCache { * @return string[] Message keys that can be passed one-by-one to get() method. */ public function getKeys() { - $value = $this->open()->get( '#keys' ); - $array = unserialize( $value ); + $reader = $this->open(); + $keys = []; - return $array; + $key = $reader->firstkey(); + while ( $key !== false ) { + if ( ( $key[0] ?? '' ) !== '#' ) { + $keys[] = $key; + } + + $key = $reader->nextkey(); + } + + return $keys; } /** @@ -109,32 +109,53 @@ class MessageGroupCache { } /** + * Get a list of authors. + * @return string[] + * @since 2020.04 + */ + public function getAuthors(): array { + $cache = $this->open(); + return $cache->exists( '#authors' ) ? + $this->unserialize( $cache->get( '#authors' ) ) : []; + } + + /** + * Get other data cached from the FFS class. + * @return array + * @since 2020.04 + */ + public function getExtra(): array { + $cache = $this->open(); + return $cache->exists( '#extra' ) ? $this->unserialize( $cache->get( '#extra' ) ) : []; + } + + /** * Populates the cache from current state of the source file. * @param bool|string $created Unix timestamp when the cache is created (for automatic updates). */ public function create( $created = false ) { $this->close(); // Close the reader instance just to be sure - $messages = $this->group->load( $this->code ); + $parseOutput = $this->group->parseExternal( $this->code ); + $messages = $parseOutput['MESSAGES']; if ( $messages === [] ) { if ( $this->exists() ) { // Delete stale cache files - unlink( $this->getCacheFileName() ); + unlink( $this->getCacheFilePath() ); } return; // Don't create empty caches } $hash = md5( file_get_contents( $this->group->getSourceFilePath( $this->code ) ) ); - wfMkdirParents( dirname( $this->getCacheFileName() ) ); - $cache = \Cdb\Writer::open( $this->getCacheFileName() ); - $keys = array_keys( $messages ); - $cache->set( '#keys', serialize( $keys ) ); + wfMkdirParents( dirname( $this->getCacheFilePath() ) ); + $cache = \Cdb\Writer::open( $this->getCacheFilePath() ); foreach ( $messages as $key => $value ) { $cache->set( $key, $value ); } - + $cache->set( '#authors', $this->serialize( $parseOutput['AUTHORS'] ) ); + $cache->set( '#extra', $this->serialize( $parseOutput['EXTRA'] ) ); $cache->set( '#created', $created ?: wfTimestamp() ); $cache->set( '#updated', wfTimestamp() ); $cache->set( '#filehash', $hash ); @@ -154,24 +175,27 @@ class MessageGroupCache { */ public function isValid( &$reason ) { $group = $this->group; - $groupId = $group->getId(); + $uniqueId = $this->getCacheFilePath(); $pattern = $group->getSourceFilePath( '*' ); $filename = $group->getSourceFilePath( $this->code ); + $parseOutput = null; + // If the file pattern is not dependent on the language, we will assume // that all translations are stored in one file. This means we need to // actually parse the file to know if a language is present. if ( strpos( $pattern, '*' ) === false ) { - $source = $group->getFFS()->read( $this->code ) !== false; + $parseOutput = $group->parseExternal( $this->code ); + $source = $parseOutput['MESSAGES'] !== []; } else { - static $globCache = null; - if ( !isset( $globCache[$groupId] ) ) { - $globCache[$groupId] = array_flip( glob( $pattern, GLOB_NOESCAPE ) ); + static $globCache = []; + if ( !isset( $globCache[$uniqueId] ) ) { + $globCache[$uniqueId] = array_flip( glob( $pattern, GLOB_NOESCAPE ) ); // Definition file might not match the above pattern - $globCache[$groupId][$group->getSourceFilePath( 'en' )] = true; + $globCache[$uniqueId][$group->getSourceFilePath( 'en' )] = true; } - $source = isset( $globCache[$groupId][$filename] ); + $source = isset( $globCache[$uniqueId][$filename] ); } $cache = $this->exists(); @@ -204,7 +228,8 @@ class MessageGroupCache { } // Message count check - $messages = $group->load( $this->code ); + $parseOutput = $parseOutput ?? $group->parseExternal( $this->code ); + $messages = $parseOutput['MESSAGES']; // CDB converts numbers to strings $count = (int)( $this->get( '#msgcount' ) ); if ( $count !== count( $messages ) ) { @@ -228,17 +253,28 @@ class MessageGroupCache { return false; } + private function serialize( array $data ): string { + // Using simple prefix for easy future extension + return 'J' . json_encode( $data ); + } + + private function unserialize( string $serialized ): array { + $type = $serialized[0]; + + if ( $type !== 'J' ) { + throw new RuntimeException( 'Unknown serialization format' ); + } + + return json_decode( substr( $serialized, 1 ), true ); + } + /** * Open the cache for reading. - * @return self + * @return \Cdb\Reader */ protected function open() { if ( $this->cache === null ) { - $this->cache = \Cdb\Reader::open( $this->getCacheFileName() ); - if ( $this->cache->get( '#version' ) !== '3' ) { - $this->close(); - unlink( $this->getCacheFileName() ); - } + $this->cache = \Cdb\Reader::open( $this->getCacheFilePath() ); } return $this->cache; @@ -258,19 +294,7 @@ class MessageGroupCache { * Returns full path to the cache file. * @return string */ - protected function getCacheFileName() { - $cacheFileName = "translate_groupcache-{$this->group->getId()}/{$this->code}.cdb"; - - return TranslateUtils::cacheFile( $cacheFileName ); - } - - /** - * Returns full path to the old cache file location. - * @return string - */ - protected function getOldCacheFileName() { - $cacheFileName = "translate_groupcache-{$this->group->getId()}-{$this->code}.cdb"; - - return TranslateUtils::cacheFile( $cacheFileName ); + protected function getCacheFilePath(): string { + return $this->cacheFilePath; } } diff --git a/MLEB/Translate/utils/MessageGroupStates.php b/MLEB/Translate/utils/MessageGroupStates.php index de20f6c8..11161356 100644 --- a/MLEB/Translate/utils/MessageGroupStates.php +++ b/MLEB/Translate/utils/MessageGroupStates.php @@ -14,7 +14,7 @@ * @since 2012-10-05 */ class MessageGroupStates { - const CONDKEY = 'state conditions'; + private const CONDKEY = 'state conditions'; protected $config; @@ -31,10 +31,6 @@ class MessageGroupStates { public function getConditions() { $conf = $this->config; - if ( isset( $conf[self::CONDKEY] ) ) { - return $conf[self::CONDKEY]; - } else { - return []; - } + return $conf[self::CONDKEY] ?? []; } } diff --git a/MLEB/Translate/utils/MessageGroupStatesUpdaterJob.php b/MLEB/Translate/utils/MessageGroupStatesUpdaterJob.php index c40bc4ec..dda4a8f9 100644 --- a/MLEB/Translate/utils/MessageGroupStatesUpdaterJob.php +++ b/MLEB/Translate/utils/MessageGroupStatesUpdaterJob.php @@ -8,6 +8,8 @@ * @license GPL-2.0-or-later */ +use MediaWiki\Extensions\Translate\SystemUsers\FuzzyBot; + /** * Logic for handling automatic message group state changes * @@ -128,7 +130,7 @@ class MessageGroupStatesUpdaterJob extends Job { */ public static function getNewState( $stats, $transitions ) { foreach ( $transitions as $transition ) { - list( $newState, $conds ) = $transition; + [ $newState, $conds ] = $transition; $match = true; foreach ( $conds as $type => $cond ) { diff --git a/MLEB/Translate/utils/MessageGroupStats.php b/MLEB/Translate/utils/MessageGroupStats.php index 2fbcf9f9..91267f22 100644 --- a/MLEB/Translate/utils/MessageGroupStats.php +++ b/MLEB/Translate/utils/MessageGroupStats.php @@ -20,17 +20,19 @@ use Wikimedia\Rdbms\IDatabase; */ class MessageGroupStats { /// Name of the database table - const TABLE = 'translate_groupstats'; + private const TABLE = 'translate_groupstats'; - const TOTAL = 0; ///< Array index - const TRANSLATED = 1; ///< Array index - const FUZZY = 2; ///< Array index - const PROOFREAD = 3; ///< Array index + public const TOTAL = 0; ///< Array index + public const TRANSLATED = 1; ///< Array index + public const FUZZY = 2; ///< Array index + public const PROOFREAD = 3; ///< Array index /// If stats are not cached, do not attempt to calculate them on the fly - const FLAG_CACHE_ONLY = 1; + public const FLAG_CACHE_ONLY = 1; /// Ignore cached values. Useful for updating stale values. - const FLAG_NO_CACHE = 2; + public const FLAG_NO_CACHE = 2; + /// Do not defer updates. Meant for jobs like MessageGroupStatsRebuildJob. + public const FLAG_IMMEDIATE_WRITES = 4; /** * @var array[] @@ -106,7 +108,14 @@ class MessageGroupStats { */ public static function forLanguage( $code, $flags = 0 ) { if ( !self::isValidLanguage( $code ) ) { - return self::getUnknownStats(); + $stats = []; + $groups = MessageGroups::singleton()->getGroups(); + $ids = array_keys( $groups ); + foreach ( $ids as $id ) { + $stats[$id] = self::getUnknownStats(); + } + + return $stats; } $stats = self::forLanguageInternal( $code, [], $flags ); @@ -129,7 +138,13 @@ class MessageGroupStats { public static function forGroup( $id, $flags = 0 ) { $group = MessageGroups::getGroup( $id ); if ( !self::isValidMessageGroup( $group ) ) { - return []; + $languages = self::getLanguages(); + $stats = []; + foreach ( $languages as $code ) { + $stats[$code] = self::getUnknownStats(); + } + + return $stats; } $stats = self::forGroupInternal( $group, [], $flags ); @@ -193,7 +208,7 @@ class MessageGroupStats { */ private static function internalClearGroups( $code, array $groups ) { $stats = []; - foreach ( $groups as $id => $group ) { + foreach ( $groups as $group ) { // $stats is modified by reference self::forItemInternal( $stats, $group, $code, 0 ); } @@ -209,7 +224,7 @@ class MessageGroupStats { * itself. * * @param string[] $ids - * @return string[] + * @return MessageGroup[] */ private static function getSortedGroupsForClearing( array $ids ) { $groups = array_map( [ MessageGroups::class, 'getGroup' ], $ids ); @@ -259,7 +274,7 @@ class MessageGroupStats { */ public static function clearAll() { $dbw = wfGetDB( DB_MASTER ); - $dbw->delete( self::TABLE, '*' ); + $dbw->delete( self::TABLE, '*', __METHOD__ ); wfDebugLog( 'messagegroupstats', 'Cleared everything :(' ); } @@ -330,7 +345,7 @@ class MessageGroupStats { * @param int $flags Combination of FLAG_* constants. * @return array[] */ - protected static function forLanguageInternal( $code, array $stats = [], $flags ) { + protected static function forLanguageInternal( $code, array $stats, $flags ) { $groups = MessageGroups::singleton()->getGroups(); $ids = array_keys( $groups ); @@ -349,7 +364,7 @@ class MessageGroupStats { /** * @param AggregateMessageGroup $agg - * @return mixed + * @return MessageGroup[] */ protected static function expandAggregates( AggregateMessageGroup $agg ) { $flattened = []; @@ -372,13 +387,13 @@ class MessageGroupStats { * @param int $flags Combination of FLAG_* constants. * @return array[] */ - protected static function forGroupInternal( MessageGroup $group, array $stats = [], $flags ) { + protected static function forGroupInternal( MessageGroup $group, array $stats, $flags ) { $id = $group->getId(); $res = self::selectRowsIdLang( [ $id ], null, $flags ); $stats = self::extractResults( $res, [ $id ], $stats ); - # Go over each language filling missing entries + // Go over each language filling missing entries $languages = self::getLanguages(); foreach ( $languages as $code ) { if ( isset( $stats[$id][$code] ) ) { @@ -398,12 +413,12 @@ class MessageGroupStats { /** * Fetch rows from the database. Use extractResults to process this value. * - * @param null|string[] $ids List of message group ids - * @param null|string[] $codes List of language codes + * @param ?string[] $ids List of message group ids + * @param ?string[] $codes List of language codes * @param int $flags Combination of FLAG_* constants. * @return Traversable Database result object */ - protected static function selectRowsIdLang( array $ids = null, array $codes = null, $flags ) { + protected static function selectRowsIdLang( ?array $ids, ?array $codes, $flags ) { if ( $flags & self::FLAG_NO_CACHE ) { return []; } @@ -489,12 +504,12 @@ class MessageGroupStats { } foreach ( $expanded as $sid => $subgroup ) { - # Discouraged groups may belong to another group, usually if there - # is an aggregate group for all translatable pages. In that case - # calculate and store the statistics, but don't count them as part of - # the aggregate group, so that the numbers in Special:LanguageStats - # add up. The statistics for discouraged groups can still be viewed - # through Special:MessageGroupStats. + // Discouraged groups may belong to another group, usually if there + // is an aggregate group for all translatable pages. In that case + // calculate and store the statistics, but don't count them as part of + // the aggregate group, so that the numbers in Special:LanguageStats + // add up. The statistics for discouraged groups can still be viewed + // through Special:MessageGroupStats. if ( !isset( $stats[$sid][$code] ) ) { $stats[$sid][$code] = self::forItemInternal( $stats, $subgroup, $code, $flags ); } @@ -532,14 +547,21 @@ class MessageGroupStats { if ( $code === $wgTranslateDocumentationLanguageCode ) { $ffs = $group->getFFS(); if ( $ffs instanceof GettextFFS ) { - $template = $ffs->read( 'en' ); - $infile = []; - foreach ( $template['TEMPLATE'] as $key => $data ) { - if ( isset( $data['comments']['.'] ) ) { - $infile[$key] = '1'; + /** + * @var FileBasedMessageGroup $group + */ + '@phan-var FileBasedMessageGroup $group'; + $cache = $group->getMessageGroupCache( $group->getSourceLanguage() ); + if ( $cache->exists() ) { + $template = $cache->getExtra()['TEMPLATE'] ?? []; + $infile = []; + foreach ( $template as $key => $data ) { + if ( isset( $data['comments']['.'] ) ) { + $infile[$key] = '1'; + } } + $collection->setInFile( $infile ); } - $collection->setInFile( $infile ); } } @@ -605,12 +627,12 @@ class MessageGroupStats { } $primaryKey = [ 'tgs_group', 'tgs_lang' ]; - $dbw->replace( $table, $primaryKey, $updates, $method ); + $dbw->replace( $table, [ $primaryKey ], $updates, $method ); $updates = []; } ); - if ( defined( 'MEDIAWIKI_JOB_RUNNER' ) ) { + if ( $flags & self::FLAG_IMMEDIATE_WRITES ) { call_user_func( $updateOp ); } else { DeferredUpdates::addCallableUpdate( $updateOp ); diff --git a/MLEB/Translate/utils/MessageGroupStatsRebuildJob.php b/MLEB/Translate/utils/MessageGroupStatsRebuildJob.php index d6d3b448..e813d614 100644 --- a/MLEB/Translate/utils/MessageGroupStatsRebuildJob.php +++ b/MLEB/Translate/utils/MessageGroupStatsRebuildJob.php @@ -8,7 +8,7 @@ */ /** - * Job for rebuilding message index. + * Job for rebuilding message group stats. * * @ingroup JobQueue */ @@ -23,6 +23,20 @@ class MessageGroupStatsRebuildJob extends Job { } /** + * Force updating of message group stats for given groups. + * + * This uses cache for groups not given. If given groups have dependencies such + * as an aggregate group and it's subgroup, this attempts to take care of it so + * that no duplicate work is done. + * + * @param string[] $messageGroupIds + * @return self + */ + public static function newRefreshGroupsJob( array $messageGroupIds ) { + return new self( Title::newMainPage(), [ 'cleargroups' => $messageGroupIds ] ); + } + + /** * @param Title $title * @param array $params */ @@ -34,15 +48,29 @@ class MessageGroupStatsRebuildJob extends Job { $params = $this->params; $flags = 0; + // Sanity check that this is run via JobQueue. Immediate writes are only safe when they + // are run in isolation, e.g. as a separate job in the JobQueue. + if ( defined( 'MEDIAWIKI_JOB_RUNNER' ) ) { + $flags |= MessageGroupStats::FLAG_IMMEDIATE_WRITES; + } + + // This is to make sure the priority value is not read from the process cache. + // There is still a possibility that, due to replication lag, an old value is read. + MessageGroups::singleton()->clearProcessCache(); + if ( isset( $params[ 'purge' ] ) && $params[ 'purge' ] ) { $flags |= MessageGroupStats::FLAG_NO_CACHE; } if ( isset( $params[ 'groupid' ] ) ) { MessageGroupStats::forGroup( $params[ 'groupid' ], $flags ); - } - if ( isset( $params[ 'languagecode' ] ) ) { - MessageGroupStats::forGroup( $params[ 'languagecode' ], $flags ); + } elseif ( isset( $params[ 'cleargroups' ] ) ) { + // clearGroup takes an array of group ids, but no flags + MessageGroupStats::clearGroup( $params[ 'cleargroups' ] ); + } elseif ( isset( $params[ 'languagecode' ] ) ) { + MessageGroupStats::forLanguage( $params[ 'languagecode' ], $flags ); + } else { + throw new InvalidArgumentException( 'No groupid or languagecode or cleargroup provided' ); } return true; diff --git a/MLEB/Translate/utils/MessageGroupWANCache.php b/MLEB/Translate/utils/MessageGroupWANCache.php new file mode 100644 index 00000000..4df5b4c7 --- /dev/null +++ b/MLEB/Translate/utils/MessageGroupWANCache.php @@ -0,0 +1,177 @@ +<?php +/** + * This file contains a wrapper around WANObjectCache + * + * @file + * @author Abijeet Patro + * @license GPL-2.0-or-later + */ + +/** + * Wrapper around WANObjectCache providing a simpler interface for + * MessageGroups to use the cache. + * @since 2019.05 + */ +class MessageGroupWANCache { + + /** + * @var WANObjectCache + */ + protected $cache; + + /** + * Cache key + * + * @var string + */ + protected $cacheKey; + + /** + * Cache version + * + * @var int + */ + protected $cacheVersion; + + /** + * To be called when the cache is empty or expired to get the data + * to repopulate the cache + * @var \Closure + */ + protected $regenerator; + + /** + * @see @https://doc.wikimedia.org/mediawiki-core/master/php/classWANObjectCache.html + * @var int + */ + protected $lockTSE; + + /** + * @see @https://doc.wikimedia.org/mediawiki-core/master/php/classWANObjectCache.html + * @var array + */ + protected $checkKeys; + + /** + * @see @https://doc.wikimedia.org/mediawiki-core/master/php/classWANObjectCache.html + * @var \Closure + */ + protected $touchedCallback; + + /** + * @see @https://doc.wikimedia.org/mediawiki-core/master/php/classWANObjectCache.html + * @var int + */ + protected $ttl; + + /** + * A prefix for all keys saved by this cache + * @var string + */ + private const KEY_PREFIX = 'translate-mg'; + + public function __construct( WANObjectCache $cache ) { + $this->cache = $cache; + } + + /** + * Fetches value from cache for a message group. + * + * @param bool $recache + * @return mixed + */ + public function getValue( $recache = false ) { + $this->checkConfig(); + + $cacheData = $this->cache->getWithSetCallback( + $this->cacheKey, + $this->ttl, + $this->regenerator, + [ + 'lockTSE' => $this->lockTSE, // avoid stampedes (mutex) + 'checkKeys' => $this->checkKeys, + 'touchedCallback' => function ( $value ) { + if ( $this->touchedCallback && call_user_func( $this->touchedCallback, $value ) ) { + // treat value as if it just expired (for "lockTSE") + return time(); + } + + return null; + }, + // "miss" on recache + 'minAsOf' => $recache ? INF : WANObjectCache::MIN_TIMESTAMP_NONE, + ] + ); + + return $cacheData; + } + + /** + * Sets value in the cache for the message group + * + * @param mixed $cacheData + */ + public function setValue( $cacheData ) { + $this->checkConfig(); + $this->cache->set( $this->cacheKey, $cacheData, $this->ttl ); + } + + public function touchKey() { + $this->checkConfig(); + $this->cache->touchCheckKey( $this->cacheKey ); + } + + /** + * Deletes the cached value + */ + public function delete() { + $this->checkConfig(); + $this->cache->delete( $this->cacheKey ); + } + + /** + * Configure the message group. This must be called before making a call to any other + * method. + * + * @param array $config + */ + public function configure( array $config ) { + $this->cacheKey = $config['key'] ?? null; + $this->cacheVersion = $config['version'] ?? null; + $this->regenerator = $config['regenerator'] ?? null; + $this->lockTSE = $config['lockTSE'] ?? 30; + $this->checkKeys = $config['checkKeys'] ?? [ $this->cacheKey ]; + $this->touchedCallback = $config['touchedCallback'] ?? null; + $this->ttl = $config['ttl'] ?? WANObjectCache::TTL_DAY; + + $this->checkConfig(); + + if ( $this->cacheVersion ) { + $this->cacheKey = $this->cache->makeKey( self::KEY_PREFIX, + strtolower( $this->cacheKey ), 'v' . $this->cacheVersion ); + } else { + $this->cacheKey = $this->cache->makeKey( + self::KEY_PREFIX, strtolower( $this->cacheKey ) + ); + } + } + + /** + * Check to see if the instance is configured properly. + */ + protected function checkConfig() { + if ( $this->cacheKey === null ) { + throw new \InvalidArgumentException( "Invalid cache key set. " . + "Ensure you have called the configure function before get / setting values." ); + } + + if ( !is_callable( $this->regenerator ) ) { + throw new \InvalidArgumentException( "Invalid regenerator set. " . + "Ensure you have called the configure function before get / setting values." ); + } + + if ( $this->touchedCallback && !is_callable( $this->touchedCallback ) ) { + throw new \InvalidArgumentException( "touchedCallback is not callable. " ); + } + } +} diff --git a/MLEB/Translate/utils/MessageHandle.php b/MLEB/Translate/utils/MessageHandle.php index 65f95bd5..bb9c43c5 100644 --- a/MLEB/Translate/utils/MessageHandle.php +++ b/MLEB/Translate/utils/MessageHandle.php @@ -7,13 +7,17 @@ * @license GPL-2.0-or-later */ +use MediaWiki\Linker\LinkTarget; +use MediaWiki\Logger\LoggerFactory; +use MediaWiki\MediaWikiServices; + /** * Class for pointing to messages, like Title class is for titles. * @since 2011-03-13 */ class MessageHandle { /** - * @var Title + * @var LinkTarget */ protected $title; @@ -32,7 +36,7 @@ class MessageHandle { */ protected $groupIds; - public function __construct( Title $title ) { + public function __construct( LinkTarget $title ) { $this->title = $title; } @@ -42,7 +46,7 @@ class MessageHandle { */ public function isMessageNamespace() { global $wgTranslateMessageNamespaces; - $namespace = $this->getTitle()->getNamespace(); + $namespace = $this->title->getNamespace(); return in_array( $namespace, $wgTranslateMessageNamespaces ); } @@ -53,9 +57,8 @@ class MessageHandle { */ public function figureMessage() { if ( $this->key === null ) { - $title = $this->getTitle(); // Check if this is a valid message first - $this->key = $title->getDBkey(); + $this->key = $this->title->getDBkey(); $known = MessageIndex::singleton()->getGroupIds( $this ) !== []; $pos = strrpos( $this->key, '/' ); @@ -99,10 +102,9 @@ class MessageHandle { * @since 2016-01 */ public function getEffectiveLanguage() { - global $wgContLang; $code = $this->getCode(); if ( $code === '' || $this->isDoc() ) { - return $wgContLang; + return MediaWikiServices::getInstance()->getContentLanguage(); } return wfGetLangObj( $code ); @@ -124,7 +126,7 @@ class MessageHandle { * @return bool */ public function isPageTranslation() { - return $this->getTitle()->inNamespace( NS_TRANSLATIONS ); + return $this->title->inNamespace( NS_TRANSLATIONS ); } /** @@ -175,10 +177,19 @@ class MessageHandle { // Do another check that the group actually exists $group = $this->getGroup(); if ( !$group ) { - $warning = "MessageIndex is out of date – refers to unknown group {$groups[0]}. "; - $warning .= 'Doing a rebuild.'; - wfWarn( $warning ); - MessageIndexRebuildJob::newJob()->run(); + $logger = LoggerFactory::getInstance( 'Translate' ); + $logger->warning( + '[MessageHandle] MessageIndex is out of date. Page {pagename} refers to ' . + 'unknown group {messagegroup}', + [ + 'pagename' => $this->getTitle()->getPrefixedText(), + 'messagegroup' => $groups[0], + ] + ); + + // Schedule a job in the job queue (with deduplication) + $job = MessageIndexRebuildJob::newJob(); + JobQueueGroup::singleton()->push( $job ); return false; } @@ -191,7 +202,7 @@ class MessageHandle { * @return Title */ public function getTitle() { - return $this->title; + return Title::newFromLinkTarget( $this->title ); } /** @@ -259,29 +270,23 @@ class MessageHandle { * @since 2017.10 */ public function getInternalKey() { - global $wgContLang; - $key = $this->getKey(); - if ( !MWNamespace::isCapitalized( $this->getTitle()->getNamespace() ) ) { + if ( !MWNamespace::isCapitalized( $this->title->getNamespace() ) ) { return $key; } $group = $this->getGroup(); - $keys = []; + $keys = $group->getKeys(); // We cannot reliably map from the database key to the internal key if // capital links setting is enabled for the namespace. - if ( method_exists( $group, 'getKeys' ) ) { - $keys = $group->getKeys(); - } else { - $keys = array_keys( $group->getDefinitions() ); - } if ( in_array( $key, $keys, true ) ) { return $key; } - $lcKey = $wgContLang->lcfirst( $key ); + $lcKey = MediaWikiServices::getInstance()->getContentLanguage() + ->lcfirst( $key ); if ( in_array( $lcKey, $keys, true ) ) { return $lcKey; } diff --git a/MLEB/Translate/utils/MessageIndex.php b/MLEB/Translate/utils/MessageIndex.php index 0015d0aa..128efe85 100644 --- a/MLEB/Translate/utils/MessageIndex.php +++ b/MLEB/Translate/utils/MessageIndex.php @@ -8,6 +8,8 @@ * @license GPL-2.0-or-later */ +use MediaWiki\Logger\LoggerFactory; + /** * Creates a database of keys in all groups, so that namespace and key can be * used to get the groups they belong to. This is used as a fallback when @@ -17,6 +19,8 @@ * to message groups. */ abstract class MessageIndex { + private const CACHEKEY = 'Translate-MessageIndex-interim'; + /** * @var self */ @@ -27,6 +31,9 @@ abstract class MessageIndex { */ private static $keysCache; + /** @var BagOStuff */ + protected $interimCache; + /** * @return self */ @@ -34,7 +41,11 @@ abstract class MessageIndex { if ( self::$instance === null ) { global $wgTranslateMessageIndex; $params = $wgTranslateMessageIndex; + if ( is_string( $params ) ) { + $params = (array)$params; + } $class = array_shift( $params ); + // @phan-suppress-next-line PhanTypeExpectedObjectOrClassName self::$instance = new $class( $params ); } @@ -57,7 +68,7 @@ abstract class MessageIndex { * @param MessageHandle $handle * @return array */ - public static function getGroupIds( MessageHandle $handle ) { + public static function getGroupIds( MessageHandle $handle ): array { global $wgTranslateMessageNamespaces; $title = $handle->getTitle(); @@ -73,10 +84,7 @@ abstract class MessageIndex { $cache = self::getCache(); $value = $cache->get( $normkey ); if ( $value === null ) { - $value = self::singleton()->get( $normkey ); - $value = $value !== null - ? (array)$value - : []; + $value = (array)self::singleton()->getWithCache( $normkey ); $cache->set( $normkey, $value ); } @@ -104,6 +112,15 @@ abstract class MessageIndex { return count( $groups ) ? array_shift( $groups ) : null; } + private function getWithCache( $key ) { + $interimCacheValue = $this->getInterimCache()->get( self::CACHEKEY ); + if ( $interimCacheValue && isset( $interimCacheValue['newKeys'][$key] ) ) { + return $interimCacheValue['newKeys'][$key]; + } + + return $this->get( $key ); + } + /** * Looks up the stored value for single key. Only for testing. * @since 2012-04-10 @@ -113,11 +130,7 @@ abstract class MessageIndex { protected function get( $key ) { // Default implementation $mi = $this->retrieve(); - if ( isset( $mi[$key] ) ) { - return $mi[$key]; - } else { - return null; - } + return $mi[$key] ?? null; } /** @@ -144,7 +157,16 @@ abstract class MessageIndex { return true; } - public function rebuild() { + /** + * Creates the index from scratch. + * + * @param float|null $timestamp Purge interim caches older than this timestamp. + * @return array + * @throws Exception + */ + public function rebuild( float $timestamp = null ): array { + $logger = LoggerFactory::getInstance( 'Translate' ); + static $recursion = 0; if ( $recursion > 0 ) { @@ -156,12 +178,24 @@ abstract class MessageIndex { } $recursion++; + $logger->info( + '[MessageIndex] Started rebuild. Initiated by {callers}', + [ 'callers' => wfGetAllCallers( 20 ) ] + ); + $groups = MessageGroups::singleton()->getGroups(); + $tsStart = microtime( true ); if ( !$this->lock() ) { - throw new Exception( __CLASS__ . ': unable to acquire lock' ); + throw new MessageIndexException( __CLASS__ . ': unable to acquire lock' ); } + $lockWaitDuration = microtime( true ) - $tsStart; + $logger->info( + '[MessageIndex] Got lock in {duration}', + [ 'duration' => $lockWaitDuration ] + ); + self::getCache()->clear(); $new = $old = []; @@ -194,6 +228,27 @@ abstract class MessageIndex { $diff = self::getArrayDiff( $old, $new ); $this->store( $new, $diff['keys'] ); $this->unlock(); + + $criticalSectionDuration = microtime( true ) - $tsStart - $lockWaitDuration; + $logger->info( + '[MessageIndex] Finished critical section in {duration}', + [ 'duration' => $criticalSectionDuration ] + ); + + $cache = $this->getInterimCache(); + $interimCacheValue = $cache->get( self::CACHEKEY ); + $timestamp = $timestamp ?? microtime( true ); + if ( $interimCacheValue ) { + if ( $interimCacheValue['timestamp'] <= $timestamp ) { + $cache->delete( self::CACHEKEY ); + } else { + // We got timestamp lower than newest front cache. This may be caused due to + // job deduplication. Just in case, spin off a new job to clean up the cache. + $job = MessageIndexRebuildJob::newJob(); + JobQueueGroup::singleton()->push( $job ); + } + } + $this->clearMessageGroupStats( $diff ); $recursion--; @@ -201,6 +256,34 @@ abstract class MessageIndex { return $new; } + private function getInterimCache(): BagOStuff { + return ObjectCache::getInstance( CACHE_ANYTHING ); + } + + public function storeInterim( MessageGroup $group, array $newKeys ): void { + $namespace = $group->getNamespace(); + $id = $group->getId(); + + $normalizedNewKeys = []; + foreach ( $newKeys as $key ) { + $normalizedNewKeys[TranslateUtils::normaliseKey( $namespace, $key )] = $id; + } + + $cache = $this->getInterimCache(); + // Merge existing with existing keys + $interimCacheValue = $cache->get( self::CACHEKEY, $cache::READ_LATEST ); + if ( $interimCacheValue ) { + $normalizedNewKeys = array_merge( $interimCacheValue['newKeys'], $normalizedNewKeys ); + } + + $value = [ + 'timestamp' => microtime( true ), + 'newKeys' => $normalizedNewKeys, + ]; + + $cache->set( self::CACHEKEY, $value, $cache::TTL_DAY ); + } + /** * Compares two associative arrays. * @@ -258,7 +341,7 @@ abstract class MessageIndex { foreach ( $old as $key => $groups ) { if ( !isset( $new[$key] ) ) { $keys['del'][$key] = [ (array)$groups, [] ]; - $record( (array)$groups, [] ); + $record( (array)$groups ); } // We already checked for diffs above } @@ -275,14 +358,15 @@ abstract class MessageIndex { * @param array $diff */ protected function clearMessageGroupStats( array $diff ) { - MessageGroupStats::clearGroup( $diff['values'] ); + $job = MessageGroupStatsRebuildJob::newRefreshGroupsJob( $diff['values'] ); + JobQueueGroup::singleton()->push( $job ); foreach ( $diff['keys'] as $keys ) { foreach ( $keys as $key => $data ) { - list( $ns, $pagename ) = explode( ':', $key, 2 ); + [ $ns, $pagename ] = explode( ':', $key, 2 ); $title = Title::makeTitle( $ns, $pagename ); $handle = new MessageHandle( $title ); - list( $oldGroups, $newGroups ) = $data; + [ $oldGroups, $newGroups ] = $data; Hooks::run( 'TranslateEventMessageMembershipChange', [ $handle, $oldGroups, $newGroups ] ); } @@ -295,20 +379,8 @@ abstract class MessageIndex { * @param bool $ignore */ protected function checkAndAdd( &$hugearray, MessageGroup $g, $ignore = false ) { - if ( method_exists( $g, 'getKeys' ) ) { - $keys = $g->getKeys(); - } else { - $messages = $g->getDefinitions(); - - if ( !is_array( $messages ) ) { - return; - } - - $keys = array_keys( $messages ); - } - + $keys = $g->getKeys(); $id = $g->getId(); - $namespace = $g->getNamespace(); foreach ( $keys as $key ) { @@ -412,10 +484,6 @@ class SerializedMessageIndex extends MessageIndex { } } -/// BC -class FileCachedMessageIndex extends SerializedMessageIndex { -} - /** * Storage on the database itself. * @@ -452,14 +520,18 @@ class DatabaseMessageIndex extends MessageIndex { // Unlock once the rows are actually unlocked to avoid deadlocks if ( !$dbw->trxLevel() ) { $dbw->unlock( 'translate-messageindex', $fname ); - } elseif ( method_exists( $dbw, 'onTransactionResolution' ) ) { // 1.28 + } elseif ( is_callable( [ $dbw, 'onTransactionResolution' ] ) ) { // 1.28 $dbw->onTransactionResolution( function () use ( $dbw, $fname ) { $dbw->unlock( 'translate-messageindex', $fname ); - } ); + }, $fname ); + } elseif ( is_callable( [ $dbw, 'onTransactionCommitOrIdle' ] ) ) { + $dbw->onTransactionCommitOrIdle( function () use ( $dbw, $fname ) { + $dbw->unlock( 'translate-messageindex', $fname ); + }, $fname ); } else { $dbw->onTransactionIdle( function () use ( $dbw, $fname ) { $dbw->unlock( 'translate-messageindex', $fname ); - } ); + }, $fname ); } return true; @@ -507,7 +579,7 @@ class DatabaseMessageIndex extends MessageIndex { foreach ( [ $diff['add'], $diff['mod'] ] as $changes ) { foreach ( $changes as $key => $data ) { - list( $old, $new ) = $data; + [ , $new ] = $data; $updates[] = [ 'tmi_key' => $key, 'tmi_value' => $this->serialize( $new ), @@ -552,7 +624,7 @@ class CachedMessageIndex extends MessageIndex { */ protected $index; - protected function __construct( array $params ) { + protected function __construct() { $this->cache = wfGetCache( CACHE_ANYTHING ); } @@ -650,11 +722,7 @@ class CDBMessageIndex extends MessageIndex { $reader = $this->getReader(); // We might have the full cache loaded if ( $this->index !== null ) { - if ( isset( $this->index[$key] ) ) { - return $this->index[$key]; - } else { - return null; - } + return $this->index[$key] ?? null; } $value = $reader->get( $key ); @@ -727,11 +795,7 @@ class HashMessageIndex extends MessageIndex { * @return mixed */ protected function get( $key ) { - if ( isset( $this->index[$key] ) ) { - return $this->index[$key]; - } else { - return null; - } + return $this->index[$key] ?? null; } protected function store( array $array, array $diff ) { diff --git a/MLEB/Translate/utils/MessageIndexException.php b/MLEB/Translate/utils/MessageIndexException.php new file mode 100644 index 00000000..b0e5c7a6 --- /dev/null +++ b/MLEB/Translate/utils/MessageIndexException.php @@ -0,0 +1,12 @@ +<?php +/** + * @file + * @author Niklas Laxstrom + * @license GPL-2.0-or-later + */ + +/** + * @since 2020.05 + */ +class MessageIndexException extends RuntimeException { +} diff --git a/MLEB/Translate/utils/MessageIndexRebuildJob.php b/MLEB/Translate/utils/MessageIndexRebuildJob.php index 2b66205f..abcb7b91 100644 --- a/MLEB/Translate/utils/MessageIndexRebuildJob.php +++ b/MLEB/Translate/utils/MessageIndexRebuildJob.php @@ -8,18 +8,20 @@ * @license GPL-2.0-or-later */ +use MediaWiki\Extensions\Translate\Jobs\GenericTranslateJob; + /** * Job for rebuilding message index. * * @ingroup JobQueue */ -class MessageIndexRebuildJob extends Job { - +class MessageIndexRebuildJob extends GenericTranslateJob { /** * @return self */ public static function newJob() { - $job = new self( Title::newMainPage() ); + $timestamp = microtime( true ); + $job = new self( Title::newMainPage(), [ 'timestamp' => $timestamp ] ); return $job; } @@ -30,19 +32,55 @@ class MessageIndexRebuildJob extends Job { */ public function __construct( $title, $params = [] ) { parent::__construct( __CLASS__, $title, $params ); + $this->removeDuplicates = true; } public function run() { - MessageIndex::singleton()->rebuild(); + // Make sure we have latest version of message groups from global cache. + // This should be pretty fast, just a few cache fetches with some post processing. + MessageGroups::singleton()->clearProcessCache(); + + // BC for existing jobs which may not have this parameter set + $timestamp = $this->getParams()['timestamp'] ?? microtime( true ); + + try { + MessageIndex::singleton()->rebuild( $timestamp ); + } catch ( MessageIndexException $e ) { + // Currently there is just one type of exception: lock wait time exceeded. + // Assuming no bugs, this is a transient issue and retry will solve it. + $this->logWarning( $e->getMessage() ); + // Try again later. See ::allowRetries + return false; + } return true; } + /** @inheritDoc */ + public function allowRetries() { + // This is the default, but added for explicitness and clarity + return true; + } + + /** @inheritDoc */ + public function getDeduplicationInfo() { + $info = parent::getDeduplicationInfo(); + // The timestamp is different for every job, so ignore it. The worst that can + // happen is that the front cache is not cleared until a future job is created. + // There is a check in MessageIndex to spawn a new job if timestamp is smaller + // than expected. + // + // Ideally we would take the latest timestamp, but it seems that the job queue + // just prevents insertion of duplicate jobs instead. + unset( $info['params']['timestamp'] ); + + return $info; + } + /** * Usually this job is fast enough to be executed immediately, * in which case having it go through jobqueue only causes problems * in installations with errant job queue processing. - * @override */ public function insertIntoJobQueue() { global $wgTranslateDelayedMessageIndexRebuild; diff --git a/MLEB/Translate/utils/MessageUpdateJob.php b/MLEB/Translate/utils/MessageUpdateJob.php index 1cc0ff6a..8d4ff35a 100644 --- a/MLEB/Translate/utils/MessageUpdateJob.php +++ b/MLEB/Translate/utils/MessageUpdateJob.php @@ -8,17 +8,56 @@ * @license GPL-2.0-or-later */ +use MediaWiki\Extensions\Translate\Jobs\GenericTranslateJob; +use MediaWiki\Extensions\Translate\SystemUsers\FuzzyBot; +use MediaWiki\Extensions\Translate\Utilities\TranslateReplaceTitle; + /** * Job for updating translation pages when translation or message definition changes. * * @ingroup JobQueue */ -class MessageUpdateJob extends Job { +class MessageUpdateJob extends GenericTranslateJob { + /** + * Create a normal message update job without a rename process + * @param Title $target + * @param string $content + * @param bool $fuzzy + * @return MessageUpdateJob + */ public static function newJob( Title $target, $content, $fuzzy = false ) { $params = [ 'content' => $content, 'fuzzy' => $fuzzy, ]; + + $job = new self( $target, $params ); + + return $job; + } + + /** + * Create a message update job containing a rename process + * @param Title $target Target message being modified + * @param string $targetStr Target string + * @param string $replacement Replacement string + * @param bool $fuzzy Whether to fuzzy the message + * @param string $content Content of the source language + * @param array $otherLangContents Content to be updated for other languages + * @return MessageUpdateJob + */ + public static function newRenameJob( + Title $target, $targetStr, $replacement, $fuzzy, $content, $otherLangContents = [] + ) { + $params = [ + 'target' => $targetStr, + 'replacement' => $replacement, + 'fuzzy' => $fuzzy, + 'rename' => 'rename', + 'content' => $content, + 'otherLangs' => $otherLangContents + ]; + $job = new self( $target, $params ); return $job; @@ -30,70 +69,213 @@ class MessageUpdateJob extends Job { */ public function __construct( $title, $params = [] ) { parent::__construct( __CLASS__, $title, $params ); - $this->params = $params; } public function run() { - global $wgTranslateDocumentationLanguageCode; - - $title = $this->title; $params = $this->params; $user = FuzzyBot::getUser(); $flags = EDIT_FORCE_BOT; + $isRename = $params['rename'] ?? false; + $isFuzzy = $params['fuzzy'] ?? false; + $otherLangs = $params['otherLangs'] ?? []; + + if ( $isRename ) { + $this->title = $this->handleRename( $params['target'], $params['replacement'], $user ); + if ( $this->title === null ) { + // There was a failure, return true, but don't proceed further. + $this->logWarning( + 'Rename process could not find the source title.', + [ + 'replacement' => $params['replacement'], + 'target' => $params['target'] + ] + ); + return true; + } + } + $title = $this->title; $wikiPage = WikiPage::factory( $title ); $summary = wfMessage( 'translate-manage-import-summary' ) ->inContentLanguage()->plain(); $content = ContentHandler::makeContent( $params['content'], $title ); - $wikiPage->doEditContent( $content, $summary, $flags, false, $user ); + $editStatus = $wikiPage->doEditContent( $content, $summary, $flags, false, $user ); + if ( !$editStatus->isOK() ) { + $this->logError( + 'Failed to update content for source message', + [ + 'content' => $content, + 'errors' => $editStatus->getErrors() + ] + ); + } - // NOTE: message documentation is excluded from fuzzying! - if ( $params['fuzzy'] ) { - $handle = new MessageHandle( $title ); - $key = $handle->getKey(); + if ( $isRename ) { + // Update other language content if present. + $this->processTranslationChanges( + $otherLangs, $params['replacement'], $params['namespace'], $summary, $flags, $user + ); + } - $languages = TranslateUtils::getLanguageNames( 'en' ); - unset( $languages[$wgTranslateDocumentationLanguageCode] ); - $languages = array_keys( $languages ); + if ( $isFuzzy ) { + $this->handleFuzzy( $title ); + } - $dbw = wfGetDB( DB_MASTER ); - $fields = [ 'page_id', 'page_latest' ]; - $conds = [ 'page_namespace' => $title->getNamespace() ]; + return true; + } - $pages = []; - foreach ( $languages as $code ) { - $otherTitle = Title::makeTitleSafe( $title->getNamespace(), "$key/$code" ); - $pages[$otherTitle->getDBkey()] = true; - } - unset( $pages[$title->getDBkey()] ); - if ( $pages === [] ) { - return true; - } + /** + * Handles renames + * @param string $target + * @param string $replacement + * @param User $user + * @return Title|null + */ + private function handleRename( $target, $replacement, User $user ) { + $newSourceTitle = null; - $conds['page_title'] = array_keys( $pages ); + $sourceMessageHandle = new MessageHandle( $this->title ); + $movableTitles = TranslateReplaceTitle::getTitlesForMove( $sourceMessageHandle, $replacement ); - $res = $dbw->select( 'page', $fields, $conds, __METHOD__ ); - $inserts = []; - foreach ( $res as $row ) { - $inserts[] = [ - 'rt_type' => RevTag::getType( 'fuzzy' ), - 'rt_page' => $row->page_id, - 'rt_revision' => $row->page_latest, - ]; + if ( $movableTitles === [] ) { + $this->logError( + 'No moveable titles found with target text.', + [ + 'title' => $this->title->getPrefixedText(), + 'replacement' => $replacement, + 'target' => $target + ] + ); + return null; + } + + $renameSummary = wfMessage( 'translate-manage-import-rename-summary' ) + ->inContentLanguage()->plain(); + + /** + * @var Title[] $movableTitles + */ + foreach ( $movableTitles as $mTitle ) { + /** + * @var Title $sourceTitle + * @var Title $replacementTitle + */ + [ $sourceTitle, $replacementTitle ] = $mTitle; + $mv = new MovePage( $sourceTitle, $replacementTitle ); + + $status = $mv->move( $user, $renameSummary, false ); + if ( !$status->isOK() ) { + $this->logError( + 'Error moving message', + [ + 'target' => $sourceTitle->getPrefixedText(), + 'replacement' => $replacementTitle->getPrefixedText(), + 'errors' => $status->getErrors() + ] + ); } - if ( $inserts === [] ) { - return true; + [ , $targetCode ] = TranslateUtils::figureMessage( $replacementTitle->getText() ); + if ( !$newSourceTitle && $sourceMessageHandle->getCode() === $targetCode ) { + $newSourceTitle = $replacementTitle; } + } - $dbw->replace( - 'revtag', - [ [ 'rt_type', 'rt_page', 'rt_revision' ] ], - $inserts, - __METHOD__ + if ( $newSourceTitle ) { + return $newSourceTitle; + } else { + // This means that the old source Title was never moved + // which is not possible but handle it. + $this->logError( + 'Source title was not in the list of moveable titles.', + [ 'title' => $this->title->getPrefixedText() ] ); } + } - return true; + /** + * Handles fuzzying. Message documentation and the source language are excluded from + * fuzzying. The source language is the identified via the $title parameter + * @param Title $title + */ + private function handleFuzzy( Title $title ) { + global $wgTranslateDocumentationLanguageCode; + $handle = new MessageHandle( $title ); + + $languages = TranslateUtils::getLanguageNames( 'en' ); + + // Don't fuzzy the message documentation + unset( $languages[$wgTranslateDocumentationLanguageCode] ); + $languages = array_keys( $languages ); + + $dbw = wfGetDB( DB_MASTER ); + $fields = [ 'page_id', 'page_latest' ]; + $conds = [ 'page_namespace' => $title->getNamespace() ]; + + $pages = []; + foreach ( $languages as $code ) { + $otherTitle = $handle->getTitleForLanguage( $code ); + $pages[$otherTitle->getDBkey()] = true; + } + + // Unset to ensure that the source language is not fuzzied + unset( $pages[$title->getDBkey()] ); + + if ( $pages === [] ) { + return; + } + + $conds['page_title'] = array_keys( $pages ); + + $res = $dbw->select( 'page', $fields, $conds, __METHOD__ ); + $inserts = []; + foreach ( $res as $row ) { + $inserts[] = [ + 'rt_type' => RevTag::getType( 'fuzzy' ), + 'rt_page' => $row->page_id, + 'rt_revision' => $row->page_latest, + ]; + } + + if ( $inserts === [] ) { + return; + } + + $dbw->replace( + 'revtag', + [ [ 'rt_type', 'rt_page', 'rt_revision' ] ], + $inserts, + __METHOD__ + ); + } + + /** + * Updates the translation unit pages in non-source languages. + * @param array $langChanges + * @param string $baseTitle + * @param int $groupNamespace + * @param string $summary + * @param int $flags + * @param User $user + */ + private function processTranslationChanges( + array $langChanges, $baseTitle, $groupNamespace, $summary, $flags, User $user + ) { + foreach ( $langChanges as $code => $contentStr ) { + $titleStr = TranslateUtils::title( $baseTitle, $code, $groupNamespace ); + $title = Title::newFromText( $titleStr, $groupNamespace ); + $wikiPage = WikiPage::factory( $title ); + $content = ContentHandler::makeContent( $contentStr, $title ); + $status = $wikiPage->doEditContent( $content, $summary, $flags, false, $user ); + if ( !$status->isOK() ) { + $this->logError( + 'Failed to update content for non-source message', + [ + 'title' => $title->getPrefixedText(), + 'errors' => $status->getErrors() + ] + ); + } + } } } diff --git a/MLEB/Translate/utils/MessageWebImporter.php b/MLEB/Translate/utils/MessageWebImporter.php index fb874dc5..308eba62 100644 --- a/MLEB/Translate/utils/MessageWebImporter.php +++ b/MLEB/Translate/utils/MessageWebImporter.php @@ -10,6 +10,10 @@ * @license GPL-2.0-or-later */ +use MediaWiki\Extensions\Translate\SystemUsers\FuzzyBot; +use MediaWiki\MediaWikiServices; +use MediaWiki\Revision\SlotRecord; + /** * Class which encapsulates message importing. It scans for changes (new, changed, deleted), * displays them in pretty way with diffs and finally executes the actions the user choices. @@ -321,7 +325,7 @@ class MessageWebImporter { foreach ( $actions as $action ) { $label = $context->msg( "translate-manage-action-$action" )->text(); $name = self::escapeNameForPHP( "action-$type-$key" ); - $id = Sanitizer::escapeId( "action-$key-$action" ); + $id = Sanitizer::escapeIdForAttribute( "action-$key-$action" ); $act[] = Xml::radioLabel( $label, $name, $action, $id, $action === $defaction ); } @@ -398,7 +402,7 @@ class MessageWebImporter { * See Article::doEdit. * @param int $editFlags Integer bitfield: see Article::doEdit * @throws MWException - * @return string Action result + * @return array Action result */ public static function doAction( $action, $group, $key, $code, $message, $comment = '', $user = null, $editFlags = 0 @@ -467,42 +471,47 @@ class MessageWebImporter { * @param string $comment * @param User $user * @param int $editFlags - * @return array|String + * @return array */ public static function doFuzzy( $title, $message, $comment, $user, $editFlags = 0 ) { $context = RequestContext::getMain(); + $services = MediaWikiServices::getInstance(); if ( !$context->getUser()->isAllowed( 'translate-manage' ) ) { - return $context->msg( 'badaccess-group0' )->text(); + return [ 'badaccess-group0' ]; } - $dbw = wfGetDB( DB_MASTER ); + // Edit with fuzzybot if there is no user. + if ( !$user ) { + $user = FuzzyBot::getUser(); + } // Work on all subpages of base title. $handle = new MessageHandle( $title ); $titleText = $handle->getKey(); - $conds = [ - 'page_namespace' => $title->getNamespace(), - 'page_latest=rev_id', - 'rev_text_id=old_id', - 'page_title' . $dbw->buildLike( "$titleText/", $dbw->anyString() ), - ]; - + $revStore = $services->getRevisionStore(); + $queryInfo = $revStore->getQueryInfo( [ 'page' ] ); + $dbw = $services->getDBLoadBalancer()->getConnectionRef( DB_MASTER ); $rows = $dbw->select( - [ 'page', 'revision', 'text' ], - [ 'page_title', 'page_namespace', 'old_text', 'old_flags' ], - $conds, - __METHOD__ + $queryInfo['tables'], + $queryInfo['fields'], + [ + 'page_namespace' => $title->getNamespace(), + 'page_latest=rev_id', + 'page_title' . $dbw->buildLike( "$titleText/", $dbw->anyString() ), + ], + __METHOD__, + [], + $queryInfo['joins'] ); - // Edit with fuzzybot if there is no user. - if ( !$user ) { - $user = FuzzyBot::getUser(); + $changed = []; + $slots = []; + if ( is_callable( [ $revStore, 'getContentBlobsForBatch' ] ) ) { + $slots = $revStore->getContentBlobsForBatch( $rows, [ SlotRecord::MAIN ] )->getValue(); } - // Process all rows. - $changed = []; foreach ( $rows as $row ) { global $wgTranslateDocumentationLanguageCode; @@ -514,9 +523,14 @@ class MessageWebImporter { ) { // Use imported text, not database text. $text = $message; + } elseif ( isset( $slots[$row->rev_id] ) ) { + $slot = $slots[$row->rev_id][SlotRecord::MAIN]; + $text = self::makeTextFuzzy( $slot->blob_data ); } else { - $text = Revision::getRevisionText( $row ); - $text = self::makeTextFuzzy( $text ); + $text = self::makeTextFuzzy( $revStore->newRevisionFromRow( $row ) + ->getContent( SlotRecord::MAIN ) + ->getNativeData() + ); } // Do actual import diff --git a/MLEB/Translate/utils/ResourceLoader.php b/MLEB/Translate/utils/PHPVariableLoader.php index 3a349897..e2e0753f 100644 --- a/MLEB/Translate/utils/ResourceLoader.php +++ b/MLEB/Translate/utils/PHPVariableLoader.php @@ -15,7 +15,7 @@ class PHPVariableLoader { * Returns a global variable from PHP file by executing the file. * @param string $_filename Path to the file. * @param string $_variable Name of the variable. - * @return mixed The variable contents or null. + * @return mixed|null The variable contents or null. */ public static function loadVariableFromPHPFile( $_filename, $_variable ) { if ( !file_exists( $_filename ) ) { @@ -23,7 +23,7 @@ class PHPVariableLoader { } else { require $_filename; - return isset( $$_variable ) ? $$_variable : null; + return $$_variable ?? null; } } } diff --git a/MLEB/Translate/utils/StatsBar.php b/MLEB/Translate/utils/StatsBar.php index df2801a2..f663e84f 100644 --- a/MLEB/Translate/utils/StatsBar.php +++ b/MLEB/Translate/utils/StatsBar.php @@ -32,7 +32,7 @@ class StatsBar { /** * @param string $group * @param string $language - * @param array[]|null $stats + * @param int[]|null $stats * * @return self */ diff --git a/MLEB/Translate/utils/StatsTable.php b/MLEB/Translate/utils/StatsTable.php index 0e9bc937..03b9d73f 100644 --- a/MLEB/Translate/utils/StatsTable.php +++ b/MLEB/Translate/utils/StatsTable.php @@ -220,7 +220,7 @@ class StatsTable { /** * Makes a nice print from plain float. - * @param number $num + * @param int|float $num * @param string $to floor or ceil * @return string Plain text */ diff --git a/MLEB/Translate/utils/ToolBox.php b/MLEB/Translate/utils/ToolBox.php deleted file mode 100644 index 7efc2980..00000000 --- a/MLEB/Translate/utils/ToolBox.php +++ /dev/null @@ -1,44 +0,0 @@ -<?php -/** - * Classes for adding extension specific toolbox menu items. - * - * @file - * @author Siebrand Mazeland - * @author Niklas Laxström - * @copyright Copyright © 2008-2010, Siebrand Mazeland, Niklas Laxström - * @license GPL-2.0-or-later - */ - -/** - * Adds extension specific context aware toolbox menu items. - */ -class TranslateToolbox { - /** - * Adds link in toolbox to Special:Prefixindex to show all other - * available translations for a message. Only shown when it - * actually is a translatable/translated message. - * - * @param BaseTemplate $baseTemplate The base skin template - * @param array &$toolbox An array of toolbox items - * - * @return bool - */ - public static function toolboxAllTranslations( $baseTemplate, &$toolbox ) { - $title = $baseTemplate->getSkin()->getTitle(); - $handle = new MessageHandle( $title ); - if ( $handle->isValid() ) { - $message = $title->getNsText() . ':' . $handle->getKey(); - $url = SpecialPage::getTitleFor( 'Translations' ) - ->getLocalURL( [ 'message' => $message ] ); - - // Add the actual toolbox entry. - $toolbox[ 'alltrans' ] = [ - 'href' => $url, - 'id' => 't-alltrans', - 'msg' => 'translate-sidebar-alltrans', - ]; - } - - return true; - } -} diff --git a/MLEB/Translate/utils/TranslateLogFormatter.php b/MLEB/Translate/utils/TranslateLogFormatter.php index 2925f2cb..39a1243e 100644 --- a/MLEB/Translate/utils/TranslateLogFormatter.php +++ b/MLEB/Translate/utils/TranslateLogFormatter.php @@ -64,7 +64,7 @@ class TranslateLogFormatter extends LogFormatter { } protected function makePageLinkWithText( - Title $title = null, $text, array $parameters = [] + ?Title $title, $text, array $parameters = [] ) { if ( !$this->plaintext ) { $link = Linker::link( $title, htmlspecialchars( $text ), [], $parameters ); diff --git a/MLEB/Translate/utils/TranslateMetadata.php b/MLEB/Translate/utils/TranslateMetadata.php index 9fe7be1a..c6d0027e 100644 --- a/MLEB/Translate/utils/TranslateMetadata.php +++ b/MLEB/Translate/utils/TranslateMetadata.php @@ -11,7 +11,32 @@ */ class TranslateMetadata { - protected static $cache; + /** @var array Map of (group => key => value) */ + private static $cache = []; + + /** + * @param string[] $groups List of translate groups + */ + public static function preloadGroups( array $groups ) { + $missing = array_keys( array_diff_key( array_flip( $groups ), self::$cache ) ); + if ( !$missing ) { + return; + } + + self::$cache += array_fill_keys( $missing, null ); // cache negatives + + $dbr = TranslateUtils::getSafeReadDB(); + $conds = count( $missing ) <= 500 ? [ 'tmd_group' => $missing ] : []; + $res = $dbr->select( + 'translate_metadata', + [ 'tmd_group', 'tmd_key', 'tmd_value' ], + $conds, + __METHOD__ + ); + foreach ( $res as $row ) { + self::$cache[$row->tmd_group][$row->tmd_key] = $row->tmd_value; + } + } /** * Get a metadata value for the given group and key. @@ -20,19 +45,24 @@ class TranslateMetadata { * @return string|bool */ public static function get( $group, $key ) { - if ( self::$cache === null ) { - $dbr = wfGetDB( DB_REPLICA ); - $res = $dbr->select( 'translate_metadata', '*', [], __METHOD__ ); - foreach ( $res as $row ) { - self::$cache[$row->tmd_group][$row->tmd_key] = $row->tmd_value; - } - } + self::preloadGroups( [ $group ] ); - if ( isset( self::$cache[$group][$key] ) ) { - return self::$cache[$group][$key]; - } + return self::$cache[$group][$key] ?? false; + } - return false; + /** + * Get a metadata value for the given group and key. + * If it does not exist, return the default value. + * @param string $group + * @param string $key + * @param string $defaultValue + * @return string + */ + public static function getWithDefaultValue( + string $group, string $key, string $defaultValue + ): string { + $value = self::get( $group, $key ); + return $value === false ? $defaultValue : $value; } /** @@ -47,7 +77,8 @@ class TranslateMetadata { $data = [ 'tmd_group' => $group, 'tmd_key' => $key, 'tmd_value' => $value ]; if ( $value === false ) { unset( $data['tmd_value'] ); - $dbw->delete( 'translate_metadata', $data ); + $dbw->delete( 'translate_metadata', $data, __METHOD__ ); + unset( self::$cache[$group][$key] ); } else { $dbw->replace( 'translate_metadata', @@ -55,9 +86,8 @@ class TranslateMetadata { $data, __METHOD__ ); + self::$cache[$group][$key] = $value; } - - self::$cache = null; } /** @@ -104,6 +134,7 @@ class TranslateMetadata { public static function deleteGroup( $groupId ) { $dbw = wfGetDB( DB_MASTER ); $conds = [ 'tmd_group' => $groupId ]; - $dbw->delete( 'translate_metadata', $conds ); + $dbw->delete( 'translate_metadata', $conds, __METHOD__ ); + self::$cache[$groupId] = null; } } diff --git a/MLEB/Translate/utils/UserToggles.php b/MLEB/Translate/utils/TranslatePreferences.php index b3205598..9e4c5512 100644 --- a/MLEB/Translate/utils/UserToggles.php +++ b/MLEB/Translate/utils/TranslatePreferences.php @@ -14,36 +14,6 @@ */ class TranslatePreferences { /** - * Add 'translate-pref-nonewsletter' preference. - * This is most probably specific to translatewiki.net. Can be enabled - * with $wgTranslateNewsletterPreference. - * @param User $user - * @param array &$preferences - * @return bool - */ - public static function onGetPreferences( User $user, array &$preferences ) { - global $wgTranslateNewsletterPreference; - - if ( !$wgTranslateNewsletterPreference ) { - return true; - } - - global $wgEnableEmail; - - // Only show if email is enabled and user has a confirmed email address. - if ( $wgEnableEmail && $user->isEmailConfirmed() ) { - // 'translate-pref-nonewsletter' is used as opt-out for - // users with a confirmed email address - $preferences['translate-nonewsletter'] = [ - 'type' => 'toggle', - 'section' => 'personal/email', - 'label-message' => 'translate-pref-nonewsletter' - ]; - - } - } - - /** * Add 'translate-editlangs' preference. * These are the languages also shown when translating. * @@ -60,7 +30,7 @@ class TranslatePreferences { $languages = Language::fetchLanguageNames(); $preferences['translate-editlangs'] = [ - 'class' => 'HTMLJsSelectToInputField', + 'class' => HTMLJsSelectToInputField::class, // prefs-translate 'section' => 'editing/translate', 'label-message' => 'translate-pref-editassistlang', @@ -78,7 +48,7 @@ class TranslatePreferences { * @return JsSelectToInput */ protected static function languageSelector() { - if ( is_callable( [ 'LanguageNames', 'getNames' ] ) ) { + if ( is_callable( [ LanguageNames::class, 'getNames' ] ) ) { $lang = RequestContext::getMain()->getLanguage(); $languages = LanguageNames::getNames( $lang->getCode(), LanguageNames::FALLBACK_NORMAL diff --git a/MLEB/Translate/utils/RcFilter.php b/MLEB/Translate/utils/TranslateRcFilter.php index 7c2334f2..7c2334f2 100644 --- a/MLEB/Translate/utils/RcFilter.php +++ b/MLEB/Translate/utils/TranslateRcFilter.php diff --git a/MLEB/Translate/utils/TranslateSandbox.php b/MLEB/Translate/utils/TranslateSandbox.php index 999c4a3e..0ed2ea18 100644 --- a/MLEB/Translate/utils/TranslateSandbox.php +++ b/MLEB/Translate/utils/TranslateSandbox.php @@ -7,9 +7,12 @@ * @license GPL-2.0-or-later */ -use MediaWiki\Auth\AuthManager; use MediaWiki\Auth\AuthenticationRequest; use MediaWiki\Auth\AuthenticationResponse; +use MediaWiki\Auth\AuthManager; +use MediaWiki\Extensions\Translate\SystemUsers\TranslateUserManager; +use MediaWiki\MediaWikiServices; +use Wikimedia\ScopedCallback; /** * Utility class for the sandbox feature of Translate. Do not try this yourself. This code makes a @@ -42,35 +45,42 @@ class TranslateSandbox { 'realname' => '', ]; - self::$userToCreate = $user; - $reqs = AuthManager::singleton()->getAuthenticationRequests( AuthManager::ACTION_CREATE ); + $permissionManager = MediaWikiServices::getInstance()->getPermissionManager(); + $creator = TranslateUserManager::getUser(); + $guard = $permissionManager->addTemporaryUserRights( $creator, 'createaccount' ); + + if ( method_exists( MediaWikiServices::class, 'getAuthManager' ) ) { + // MediaWiki 1.35+ + $authManager = MediaWikiServices::getInstance()->getAuthManager(); + } else { + $authManager = AuthManager::singleton(); + } + $reqs = $authManager->getAuthenticationRequests( AuthManager::ACTION_CREATE ); $reqs = AuthenticationRequest::loadRequestsFromSubmission( $reqs, $data ); - $res = AuthManager::singleton()->beginAccountCreation( $user, $reqs, 'null:' ); - self::$userToCreate = null; + $res = $authManager->beginAccountCreation( $creator, $reqs, 'null:' ); + + ScopedCallback::consume( $guard ); switch ( $res->status ) { - case AuthenticationResponse::PASS: - break; - case AuthenticationResponse::FAIL: - // Unless things are misconfigured, this will handle errors such as username taken, - // invalid user name or too short password. The WebAPI is prechecking these to - // provide nicer error messages. - $reason = $res->message->inLanguage( 'en' )->useDatabase( false )->text(); - throw new MWException( "Account creation failed: $reason" ); - default: - // Just in case it was a Secondary that failed - $user->clearInstanceCache( 'name' ); - if ( $user->getId() ) { - self::deleteUser( $user, 'force' ); - } - throw new MWException( - 'AuthManager does not support such simplified account creation' - ); - } + case AuthenticationResponse::PASS: + break; + case AuthenticationResponse::FAIL: + // Unless things are misconfigured, this will handle errors such as username taken, + // invalid user name or too short password. The WebAPI is prechecking these to + // provide nicer error messages. + $reason = $res->message->inLanguage( 'en' )->useDatabase( false )->text(); + throw new MWException( "Account creation failed: $reason" ); + default: + // A provider requested further user input. Abort but clean up first if it was a + // secondary provider (in which case the user was created). + if ( $user->getId() ) { + self::deleteUser( $user, 'force' ); + } - // User now has an id, but we must clear the cache to see it. Without this the group - // addition below would not be saved in the database. - $user->clearInstanceCache( 'name' ); + throw new MWException( + 'AuthManager does not support such simplified account creation' + ); + } // group-translate-sandboxed group-translate-sandboxed-member $user->addGroup( 'translate-sandboxed' ); @@ -87,7 +97,6 @@ class TranslateSandbox { */ public static function deleteUser( User $user, $force = '' ) { $uid = $user->getId(); - $username = $user->getName(); if ( $force !== 'force' && !self::isSandboxed( $user ) ) { throw new MWException( 'Not a sandboxed user' ); @@ -99,20 +108,15 @@ class TranslateSandbox { $dbw->delete( 'user_groups', [ 'ug_user' => $uid ], __METHOD__ ); $dbw->delete( 'user_properties', [ 'up_user' => $uid ], __METHOD__ ); - if ( class_exists( ActorMigration::class ) ) { - $m = ActorMigration::newMigration(); + $m = ActorMigration::newMigration(); + $dbw->delete( 'actor', [ 'actor_user' => $uid ], __METHOD__ ); + // Assume no joins are needed for logging or recentchanges + $dbw->delete( 'logging', $m->getWhere( $dbw, 'log_user', $user )['conds'], __METHOD__ ); + $dbw->delete( 'recentchanges', $m->getWhere( $dbw, 'rc_user', $user )['conds'], __METHOD__ ); - // Assume no joins are needed for logging or recentchanges - $dbw->delete( 'logging', $m->getWhere( $dbw, 'log_user', $user )['conds'], __METHOD__ ); - $dbw->delete( 'recentchanges', $m->getWhere( $dbw, 'rc_user', $user )['conds'], __METHOD__ ); - } else { - $dbw->delete( 'logging', [ 'log_user' => $uid ], __METHOD__ ); - $dbw->delete( - 'recentchanges', - [ 'rc_user' => $uid, 'rc_user_text' => $username ], - __METHOD__ - ); - } + // Update the site stats + $statsUpdate = SiteStatsUpdate::factory( [ 'users' => -1 ] ); + $statsUpdate->doUpdate(); // If someone tries to access still object still, they will get anon user // data. @@ -139,15 +143,7 @@ class TranslateSandbox { */ public static function getUsers() { $dbw = TranslateUtils::getSafeReadDB(); - if ( is_callable( [ User::class, 'getQueryInfo' ] ) ) { - $userQuery = User::getQueryInfo(); - } else { - $userQuery = [ - 'tables' => [ 'user' ], - 'fields' => User::selectFields(), - 'joins' => [], - ]; - } + $userQuery = User::getQueryInfo(); $tables = array_merge( $userQuery['tables'], [ 'user_groups' ] ); $fields = $userQuery['fields']; $conds = [ @@ -290,13 +286,6 @@ class TranslateSandbox { return false; } - /// Hook: UserGetRights - public static function allowAccountCreation( $user, &$rights ) { - if ( self::$userToCreate && $user->equals( self::$userToCreate ) ) { - $rights[] = 'createaccount'; - } - } - /// Hook: onGetPreferences public static function onGetPreferences( $user, &$preferences ) { $preferences['translate-sandbox'] = $preferences['translate-sandbox-reminders'] = @@ -325,10 +314,6 @@ class TranslateSandbox { $class = get_class( $module ); if ( $module->isWriteMode() && !in_array( $class, $whitelist, true ) ) { $message = ApiMessage::create( 'apierror-writeapidenied' ); - if ( $message->getApiCode() === 'apierror-writeapidenied' ) { - // Backwards compatibility for pre-1.29 MediaWiki - $message = 'writerequired'; - } return false; } } diff --git a/MLEB/Translate/utils/TranslateSandboxEmailJob.php b/MLEB/Translate/utils/TranslateSandboxEmailJob.php index 955e7156..4c3716b9 100644 --- a/MLEB/Translate/utils/TranslateSandboxEmailJob.php +++ b/MLEB/Translate/utils/TranslateSandboxEmailJob.php @@ -1,6 +1,9 @@ <?php +use MediaWiki\MediaWikiServices; + class TranslateSandboxEmailJob extends Job { + /** * @param array $params * @return self @@ -18,13 +21,27 @@ class TranslateSandboxEmailJob extends Job { } public function run() { - $status = UserMailer::send( - $this->params['to'], - $this->params['from'], - $this->params['subj'], - $this->params['body'], - [ 'replyTo' => $this->params['replyto'] ] - ); + $services = MediaWikiServices::getInstance(); + if ( is_callable( [ $services, 'getEmailer' ] ) ) { + $status = $services + ->getEmailer() + ->send( + [ $this->params['to'] ], + $this->params['from'], + $this->params['subj'], + $this->params['body'], + null, + [ 'replyTo' => $this->params['replyto'] ] + ); + } else { + $status = UserMailer::send( + $this->params['to'], + $this->params['from'], + $this->params['subj'], + $this->params['body'], + [ 'replyTo' => $this->params['replyto'] ] + ); + } $isOK = $status->isOK(); diff --git a/MLEB/Translate/utils/TranslateToolbox.php b/MLEB/Translate/utils/TranslateToolbox.php new file mode 100644 index 00000000..0836f485 --- /dev/null +++ b/MLEB/Translate/utils/TranslateToolbox.php @@ -0,0 +1,74 @@ +<?php +/** + * Classes for adding extension specific toolbox menu items. + * + * @file + * @author Siebrand Mazeland + * @author Niklas Laxström + * @copyright Copyright © 2008-2010, Siebrand Mazeland, Niklas Laxström + * @license GPL-2.0-or-later + */ + +/** + * Adds toolbox menu item to Special:Prefixindex to show all other + * available translations for a message. Only shown when it + * actually is a translatable/translated message. + */ +class TranslateToolbox { + /** + * This handler will be called for MW < 1.35 + * + * @param BaseTemplate $baseTemplate The base skin template + * @param array &$toolbox An array of toolbox items + * + * @return void + */ + public static function toolboxAllTranslationsOld( + BaseTemplate $baseTemplate, array &$toolbox + ): void { + $skin = $baseTemplate->getSkin(); + $title = $skin->getTitle(); + $handle = new MessageHandle( $title ); + + if ( !$handle->isValid() ) { + return; + } + + $message = $title->getNsText() . ':' . $handle->getKey(); + $url = $skin::makeSpecialUrl( 'Translations', [ 'message' => $message ] ); + + // Add the actual toolbox entry. + $toolbox[ 'alltrans' ] = [ + 'href' => $url, + 'id' => 't-alltrans', + 'msg' => 'translate-sidebar-alltrans', + ]; + } + + /** + * This handler will be called for MW >= 1.35 + * + * @param Skin $skin The skin + * @param array &$sidebar Array with sidebar items + * + * @return void + */ + public static function toolboxAllTranslations( Skin $skin, array &$sidebar ): void { + $title = $skin->getTitle(); + $handle = new MessageHandle( $title ); + + if ( !$handle->isValid() ) { + return; + } + + $message = $title->getNsText() . ':' . $handle->getKey(); + $url = $skin::makeSpecialUrl( 'Translations', [ 'message' => $message ] ); + + // Add the actual toolbox entry. + $sidebar['TOOLBOX'][ 'alltrans' ] = [ + 'href' => $url, + 'id' => 't-alltrans', + 'msg' => 'translate-sidebar-alltrans', + ]; + } +} diff --git a/MLEB/Translate/utils/TranslateYaml.php b/MLEB/Translate/utils/TranslateYaml.php index 699676a9..7209c214 100644 --- a/MLEB/Translate/utils/TranslateYaml.php +++ b/MLEB/Translate/utils/TranslateYaml.php @@ -25,14 +25,8 @@ class TranslateYaml { switch ( $wgTranslateYamlLibrary ) { case 'phpyaml': // Harden: do not support unserializing objects. - // Method 1: PHP ini setting (not supported by HHVM) - // Method 2: Callback handler for !php/object $previousValue = ini_set( 'yaml.decode_php', false ); - $ignored = 0; - $callback = function ( $value ) { - return $value; - }; - $ret = yaml_parse( $text, 0, $ignored, [ '!php/object' => $callback ] ); + $ret = yaml_parse( $text ); ini_set( 'yaml.decode_php', $previousValue ); if ( $ret === false ) { // Convert failures to exceptions diff --git a/MLEB/Translate/utils/TranslationHelpers.php b/MLEB/Translate/utils/TranslationHelpers.php index 1551a1a8..9b585c72 100644 --- a/MLEB/Translate/utils/TranslationHelpers.php +++ b/MLEB/Translate/utils/TranslationHelpers.php @@ -27,7 +27,7 @@ class TranslationHelpers { /** * The group object of the message (or null if there isn't any) - * @var MessageGroup + * @var MessageGroup|null */ protected $group; @@ -87,7 +87,7 @@ class TranslationHelpers { /** * Sets the HTML id of the text area that contains the translation. - * @param String $id + * @param string $id */ public function setTextareaId( $id ) { $this->textareaId = $id; @@ -103,7 +103,7 @@ class TranslationHelpers { /** * Gets the message definition. - * @return String + * @return string */ public function getDefinition() { $this->mustBeKnownMessage(); @@ -253,7 +253,7 @@ class TranslationHelpers { $sl = Language::factory( $this->group->getSourceLanguage() ); $dialogID = $this->dialogID(); - $id = Sanitizer::escapeId( "def-$dialogID" ); + $id = Sanitizer::escapeIdForAttribute( "def-$dialogID" ); $msg = $this->adder( $id, $sl ) . "\n" . Html::rawElement( 'div', [ 'class' => 'mw-translate-edit-deftext', @@ -310,7 +310,7 @@ class TranslationHelpers { } $dialogID = $this->dialogID(); - $id = Sanitizer::escapeId( "other-$fbcode-$dialogID" ); + $id = Sanitizer::escapeIdForAttribute( "other-$fbcode-$dialogID" ); $params = [ 'class' => 'mw-translate-edit-item' ]; @@ -380,9 +380,7 @@ class TranslationHelpers { } $class .= ' mw-sp-translate-message-documentation'; - $contents = TranslateUtils::parseInlineAsInterface( - $context->getOutput(), $info - ); + $contents = $context->getOutput()->parseInlineAsInterface( $info ); return TranslateUtils::fieldset( $context->msg( 'translate-edit-information' )->rawParams( $edit )->escaped(), diff --git a/MLEB/Translate/utils/TuxMessageTable.php b/MLEB/Translate/utils/TuxMessageTable.php index 0585ac84..3a6950ce 100644 --- a/MLEB/Translate/utils/TuxMessageTable.php +++ b/MLEB/Translate/utils/TuxMessageTable.php @@ -7,7 +7,11 @@ class TuxMessageTable extends ContextSource { public function __construct( IContextSource $context, MessageGroup $group, $language ) { $this->setContext( $context ); $this->group = $group; - $this->language = $language; + if ( Language::isKnownLanguageTag( $language ) ) { + $this->language = $language; + } else { + $this->language = $context->getLanguage()->getCode(); + } } public function fullTable() { @@ -50,7 +54,7 @@ HTML; <div class="tux-action-bar hide row"> <div class="three columns tux-message-list-statsbar" data-messagegroup="$groupId"></div> <div class="three columns text-center"> - <button class="toggle button tux-proofread-own-translations-button hide-own hide"> + <button class="toggle button tux-proofread-own-translations-button hide"> $hideOwn </button> <button class="toggle button tux-editor-clear-translated hide">$clearTranslated</button> diff --git a/MLEB/Translate/utils/lc.php b/MLEB/Translate/utils/lc.php new file mode 100644 index 00000000..a41f5898 --- /dev/null +++ b/MLEB/Translate/utils/lc.php @@ -0,0 +1,42 @@ +<?php + +/** + * @file + * @author Niklas Laxström + * @license GPL-2.0-or-later + */ + +/** + * Helper function for adding namespace for message groups. + * + * It defines constants for the namespace (and talk namespace) and sets up + * restrictions and some other configuration. + * @param int $id Namespace number + * @param string $name Name of the namespace + * @param string|null $constant (optional) name of namespace constant, defaults to + * NS_ followed by upper case version of $name, e.g., NS_MEDIAWIKI + */ +function wfAddNamespace( $id, $name, $constant = null ) { + global $wgExtraNamespaces, $wgContentNamespaces, $wgTranslateMessageNamespaces, + $wgNamespaceProtection, $wgNamespacesWithSubpages, $wgNamespacesToBeSearchedDefault; + + if ( $constant === null ) { + $constant = strtoupper( "NS_$name" ); + } + + define( $constant, $id ); + define( $constant . '_TALK', $id + 1 ); + + $wgExtraNamespaces[$id] = $name; + $wgExtraNamespaces[$id + 1] = $name . '_talk'; + + $wgContentNamespaces[] = $id; + $wgTranslateMessageNamespaces[] = $id; + + $wgNamespacesWithSubpages[$id] = true; + $wgNamespacesWithSubpages[$id + 1] = true; + + $wgNamespaceProtection[$id] = [ 'translate' ]; + + $wgNamespacesToBeSearchedDefault[$id] = true; +} |