Added script to extract translations into PO files

a57081a4 · Eliot Berriot · 18e8e4fa · a57081a4 · a57081a4 · a57081a4
Verified Commit a57081a4 authored Apr 15, 2018 by Eliot Berriot
--- a/README.rst
+++ b/README.rst
@@ -208,6 +208,17 @@ Typical workflow for a merge request
 8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute!


+Internationalization
+--------------------
+
+When working on the front-end, any end-user string should be translated
+using either ``<i18next path="yourstring">`` or the ``$t('yourstring')``
+function.
+
+Extraction is done by calling ``yarn run i18n-extract``, which
+will pull all the strings from source files and put them in a PO file.
+
+
 Working with federation locally
 -------------------------------

@@ -245,7 +256,7 @@ Run a reverse proxy for your instances
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


-Crete docker network
+Create docker network
 ^^^^^^^^^^^^^^^^^^^^

 Create the federation network::
@@ -265,7 +276,7 @@ need::
    export COMPOSE_PROJECT_NAME=node2
    docker-compose -f dev.yml run --rm api python manage.py migrate
    docker-compose -f dev.yml run --rm api python manage.py createsuperuser
-    docker-compose -f dev.yml up nginx api front
+    docker-compose -f dev.yml up nginx api front nginx api celeryworker

 Note that by default, if you don't export the COMPOSE_PROJECT_NAME,
 we will default to node1 as the name of your instance.

--- a/changes/changelog.d/162-script.enhancement
+++ b/changes/changelog.d/162-script.enhancement
+Added a i18n-extract yarn script to extract strings to PO files (#162)
--- a/front/package.json
+++ b/front/package.json
@@ -8,6 +8,7 @@
    "dev": "node build/dev-server.js",
    "start": "node build/dev-server.js",
    "build": "node build/build.js",
+    "i18n-extract": "find src/ -name '*.vue' | xargs vendor/vue-i18n-xgettext/index.js",
    "unit": "cross-env BABEL_ENV=test karma start test/unit/karma.conf.js --single-run",
    "unit-watch": "cross-env BABEL_ENV=test karma start test/unit/karma.conf.js",
    "e2e": "node test/e2e/runner.js",
@@ -102,6 +103,7 @@
    "sinon-chai": "^2.8.0",
    "sinon-stub-promise": "^4.0.0",
    "url-loader": "^0.5.8",
+    "vue-i18n-xgettext": "^0.0.4",
    "vue-loader": "^12.1.0",
    "vue-style-loader": "^3.0.1",
    "vue-template-compiler": "^2.3.3",

--- a/front/vendor/vue-i18n-xgettext/extractor.js
+++ b/front/vendor/vue-i18n-xgettext/extractor.js
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };
+
+var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
+
+var _cheerio = require('cheerio');
+
+var _cheerio2 = _interopRequireDefault(_cheerio);
+
+var _pofile = require('pofile');
+
+var _pofile2 = _interopRequireDefault(_pofile);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
+var tRegexp = new RegExp('.*\\$t\\([\'\"\`](.*)[\'\"\`]\\).*', 'g');
+
+var Translation = function () {
+  function Translation(filename, lineNumber, msg) {
+    _classCallCheck(this, Translation);
+
+    this.filename = filename;
+    this.lineNumber = lineNumber;
+    this.msg = msg;
+  }
+
+  _createClass(Translation, [{
+    key: 'toPofileItem',
+    value: function toPofileItem() {
+      var item = new _pofile2.default.Item();
+      item.msgid = this.msg;
+      item.msgctxt = null;
+      item.references = [this.filename + ':' + this.lineNumber];
+      item.msgid_plural = null;
+      item.msgstr = [];
+      item.extractedComments = [];
+      return item;
+    }
+  }]);
+
+  return Translation;
+}();
+
+var Extractor = function () {
+  function Extractor(options) {
+    _classCallCheck(this, Extractor);
+
+    this.options = _extends({
+      startDelim: '{{',
+      endDelim: '}}',
+      attributes: ['path']
+    }, options);
+    this.translations = [];
+  }
+
+  _createClass(Extractor, [{
+    key: 'parse',
+    value: function parse(filename, content) {
+      var $ = _cheerio2.default.load(content, {
+        decodeEntities: false,
+        withStartIndices: true
+      });
+
+      var translations = $('template *').map(function (i, el) {
+        var node = $(el);
+        var msg = null;
+        if (node['0'].name === 'i18next') {
+          // here, we extract the translations from <i18next path="string">
+          msg = this.extractTranslationMessageFromI18Next(node);
+        }
+        if (msg) {
+          var truncatedText = content.substr(0, el.startIndex);
+          var lineNumber = truncatedText.split(/\r\n|\r|\n/).length;
+          return new Translation(filename, lineNumber, msg);
+        }
+      }.bind(this)).get();
+      var scriptTranslations = $('script,template').map(function (i, el) {
+        // here, we extract the translations from $t('string')
+        // within scripts and templates
+        var script = $(el).text();
+        var lines = script.split('\n');
+        var _translations = [];
+        lines.forEach(function (line) {
+          var truncatedText = content.substr(0, el.startIndex);
+          var matches;
+          while ((matches = tRegexp.exec(line)) !== null) {
+            var lineNumber = truncatedText.split(/\r\n|\r|\n/).length;
+            _translations.push(new Translation(filename, lineNumber, matches[1]));
+          }
+        })
+        return _translations
+      }.bind(this)).get();
+      this.translations = this.translations.concat(translations);
+      this.translations = this.translations.concat(scriptTranslations);
+    }
+  }, {
+    key: 'extractTranslationMessageFromI18Next',
+    value: function extractTranslationMessageFromI18Next(node) {
+      // extract from attributes
+      var _iteratorNormalCompletion = true;
+      var _didIteratorError = false;
+      var _iteratorError = undefined;
+
+      try {
+        for (var _iterator = this.options.attributes[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
+          var attr = _step.value;
+
+          if (node.attr('path')) {
+            return node.attr('path');
+          }
+        }
+      } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+      } finally {
+        try {
+          if (!_iteratorNormalCompletion && _iterator.return) {
+            _iterator.return();
+          }
+        } finally {
+          if (_didIteratorError) {
+            throw _iteratorError;
+          }
+        }
+      }
+    }
+  }, {
+    key: 'toPofile',
+    value: function toPofile() {
+      var pofile = new _pofile2.default();
+      pofile.headers = {
+        'Last-Translator': 'vue-i18n-xgettext',
+        'Content-Type': 'text/plain; charset=UTF-8',
+        'Content-Transfer-Encoding': '8bit',
+        'MIME-Version': '1.1'
+      };
+
+      var itemMapping = {};
+      var _iteratorNormalCompletion2 = true;
+      var _didIteratorError2 = false;
+      var _iteratorError2 = undefined;
+
+      try {
+        for (var _iterator2 = this.translations[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true) {
+          var translation = _step2.value;
+
+          var _item = translation.toPofileItem();
+          if (!itemMapping[_item.msgid]) {
+            itemMapping[_item.msgid] = _item;
+          } else {
+            var oldItem = itemMapping[_item.msgid];
+            // TODO: deal with plurals/context
+            if (_item.references.length && oldItem.references.indexOf(_item.references[0]) === -1) {
+              oldItem.references.push(_item.references[0]);
+            }
+            if (_item.extractedComments.length && soldItem.extractedComments.indexOf(_item.extractedComments[0]) === -1) {
+              oldItem.extractedComments.push(_item.extractedComments[0]);
+            }
+          }
+        }
+      } catch (err) {
+        _didIteratorError2 = true;
+        _iteratorError2 = err;
+      } finally {
+        try {
+          if (!_iteratorNormalCompletion2 && _iterator2.return) {
+            _iterator2.return();
+          }
+        } finally {
+          if (_didIteratorError2) {
+            throw _iteratorError2;
+          }
+        }
+      }
+
+      for (var msgid in itemMapping) {
+        var item = itemMapping[msgid];
+        pofile.items.push(item);
+      }
+
+      pofile.items.sort(function (a, b) {
+        return a.msgid.localeCompare(b.msgid);
+      });
+      return pofile;
+    }
+  }, {
+    key: 'toString',
+    value: function toString() {
+      return this.toPofile().toString();
+    }
+  }]);
+
+  return Extractor;
+}();
+
+exports.default = Extractor;
+//# sourceMappingURL=extractor.js.map
--- a/front/vendor/vue-i18n-xgettext/index.js
+++ b/front/vendor/vue-i18n-xgettext/index.js
+#!/usr/bin/env node
+'use strict';
+
+var _fs = require('fs');
+
+var _fs2 = _interopRequireDefault(_fs);
+
+var _minimist = require('minimist');
+
+var _minimist2 = _interopRequireDefault(_minimist);
+
+var _extractor = require('./extractor.js');
+
+var _extractor2 = _interopRequireDefault(_extractor);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+var argv = (0, _minimist2.default)(process.argv.slice(2));
+var files = argv._.sort() || [];
+var attributes = argv.attribute || [];
+var outputFile = argv.output || null;
+
+if (!files || files.length === 0) {
+  console.log('Usage: vue-i18n-xgettext [--attribute ATTRIBUTE] [--output OUTPUT_FILE] FILES');
+  process.exit(1);
+}
+
+var defaultAttributes = ['v-text'];
+var finalAttributes = defaultAttributes;
+if (typeof attributes === 'string') {
+  finalAttributes.push(attributes);
+} else {
+  finalAttributes = finalAttributes.concat(attributes);
+}
+
+var extractor = new _extractor2.default({
+  attributes: finalAttributes
+});
+
+files.forEach(function (filename) {
+  var extension = filename.split('.').pop();
+  if (extension !== 'vue') {
+    console.log('file ' + filename + ' with extension ' + extension + ' will not be processed (skipped)');
+    return;
+  }
+
+  var data = _fs2.default.readFileSync(filename, { encoding: 'utf-8' }).toString();
+
+  try {
+    extractor.parse(filename, data);
+  } catch (e) {
+    console.trace(e);
+    process.exit(1);
+  }
+});
+
+var output = extractor.toString();
+if (outputFile) {
+  _fs2.default.writeFileSync(outputFile, output);
+} else {
+  console.log(output);
+}
+//# sourceMappingURL=index.js.map
--- a/front/yarn.lock
+++ b/front/yarn.lock
@@ -1311,6 +1311,27 @@ check-types@^7.3.0:
  version "7.3.0"
  resolved "https://registry.yarnpkg.com/check-types/-/check-types-7.3.0.tgz#468f571a4435c24248f5fd0cb0e8d87c3c341e7d"

+cheerio@^0.22.0:
+  version "0.22.0"
+  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e"
+  dependencies:
+    css-select "~1.2.0"
+    dom-serializer "~0.1.0"
+    entities "~1.1.1"
+    htmlparser2 "^3.9.1"
+    lodash.assignin "^4.0.9"
+    lodash.bind "^4.1.4"
+    lodash.defaults "^4.0.1"
+    lodash.filter "^4.4.0"
+    lodash.flatten "^4.2.0"
+    lodash.foreach "^4.3.0"
+    lodash.map "^4.4.0"
+    lodash.merge "^4.4.0"
+    lodash.pick "^4.2.1"
+    lodash.reduce "^4.4.0"
+    lodash.reject "^4.4.0"
+    lodash.some "^4.4.0"
+
 chokidar@^1.4.1:
  version "1.7.0"
  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468"
@@ -1783,7 +1804,7 @@ css-loader@^0.28.0:
    postcss-value-parser "^3.3.0"
    source-list-map "^2.0.0"

-css-select@^1.1.0:
+css-select@^1.1.0, css-select@~1.2.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
  dependencies:
@@ -2110,7 +2131,7 @@ dom-serialize@^2.2.0:
    extend "^3.0.0"
    void-elements "^2.0.0"

-dom-serializer@0:
+dom-serializer@0, dom-serializer@~0.1.0:
  version "0.1.0"
  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82"
  dependencies:
@@ -2972,6 +2993,10 @@ fs.realpath@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"

+fs@0.0.1-security:
+  version "0.0.1-security"
+  resolved "https://registry.yarnpkg.com/fs/-/fs-0.0.1-security.tgz#8a7bd37186b6dddf3813f23858b57ecaaf5e41d4"
+
 fsevents@^1.0.0, fsevents@^1.1.2:
  version "1.1.3"
  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.1.3.tgz#11f82318f5fe7bb2cd22965a108e9306208216d8"
@@ -3433,7 +3458,7 @@ html-webpack-plugin@^2.28.0:
    pretty-error "^2.0.2"
    toposort "^1.0.0"

-htmlparser2@^3.8.2:
+htmlparser2@^3.8.2, htmlparser2@^3.9.1:
  version "3.9.2"
  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338"
  dependencies:
@@ -3523,10 +3548,6 @@ https-proxy-agent@1:
    debug "2"
    extend "3"

-i18next-browser-languagedetector@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/i18next-browser-languagedetector/-/i18next-browser-languagedetector-2.2.0.tgz#5f41abe61964a56dce70102ab31c3ed5d5866edc"
-
 i18next-conv@^6.0.0:
  version "6.0.0"
  resolved "https://registry.yarnpkg.com/i18next-conv/-/i18next-conv-6.0.0.tgz#875a27bfb069db894f7b0a1484e0052100bc9383"
@@ -4343,6 +4364,14 @@ lodash.assign@^4.2.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"

+lodash.assignin@^4.0.9:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.assignin/-/lodash.assignin-4.2.0.tgz#ba8df5fb841eb0a3e8044232b0e263a8dc6a28a2"
+
+lodash.bind@^4.1.4:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35"
+
 lodash.camelcase@^4.3.0:
  version "4.3.0"
  resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
@@ -4367,6 +4396,10 @@ lodash.create@3.1.1:
    lodash._basecreate "^3.0.0"
    lodash._isiterateecall "^3.0.0"

+lodash.defaults@^4.0.1:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+
 lodash.defaultsdeep@4.3.2:
  version "4.3.2"
  resolved "https://registry.yarnpkg.com/lodash.defaultsdeep/-/lodash.defaultsdeep-4.3.2.tgz#6c1a586e6c5647b0e64e2d798141b8836158be8a"
@@ -4378,6 +4411,18 @@ lodash.defaultsdeep@4.3.2:
    lodash.mergewith "^4.0.0"
    lodash.rest "^4.0.0"

+lodash.filter@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
+
+lodash.flatten@^4.2.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+
+lodash.foreach@^4.3.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
+
 lodash.get@^4.4.2:
  version "4.4.2"
  resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
@@ -4406,18 +4451,42 @@ lodash.keysin@^4.0.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/lodash.keysin/-/lodash.keysin-4.2.0.tgz#8cc3fb35c2d94acc443a1863e02fa40799ea6f28"

+lodash.map@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
+
 lodash.memoize@^4.1.2:
  version "4.1.2"
  resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"

+lodash.merge@^4.4.0:
+  version "4.6.1"
+  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.1.tgz#adc25d9cb99b9391c59624f379fbba60d7111d54"
+
 lodash.mergewith@^4.0.0, lodash.mergewith@^4.6.0:
  version "4.6.1"
  resolved "https://registry.yarnpkg.com/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz#639057e726c3afbdb3e7d42741caa8d6e4335927"

+lodash.pick@^4.2.1:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
+
+lodash.reduce@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b"
+
+lodash.reject@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415"
+
 lodash.rest@^4.0.0:
  version "4.0.5"
  resolved "https://registry.yarnpkg.com/lodash.rest/-/lodash.rest-4.0.5.tgz#954ef75049262038c96d1fc98b28fdaf9f0772aa"

+lodash.some@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d"
+
 lodash.tail@^4.1.1:
  version "4.1.1"
  resolved "https://registry.yarnpkg.com/lodash.tail/-/lodash.tail-4.1.1.tgz#d2333a36d9e7717c8ad2f7cacafec7c32b444664"
@@ -5437,6 +5506,10 @@ pluralize@^1.2.1:
  version "1.2.1"
  resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-1.2.1.tgz#d1a21483fd22bb41e58a12fa3421823140897c45"

+pofile@^1.0.2:
+  version "1.0.10"
+  resolved "https://registry.yarnpkg.com/pofile/-/pofile-1.0.10.tgz#503dda9499403984e83ff4489ba2d80af276172a"
+
 posix-character-classes@^0.1.0:
  version "0.1.1"
  resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
@@ -7292,6 +7365,15 @@ vue-hot-reload-api@^2.1.0:
  version "2.3.0"
  resolved "https://registry.yarnpkg.com/vue-hot-reload-api/-/vue-hot-reload-api-2.3.0.tgz#97976142405d13d8efae154749e88c4e358cf926"

+vue-i18n-xgettext@^0.0.4:
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/vue-i18n-xgettext/-/vue-i18n-xgettext-0.0.4.tgz#80ad654e65fb33bb5fcbd96f338f55605ab1a06f"
+  dependencies:
+    cheerio "^0.22.0"
+    fs "0.0.1-security"
+    minimist "^1.2.0"
+    pofile "^1.0.2"
+
 vue-lazyload@^1.1.4:
  version "1.2.2"
  resolved "https://registry.yarnpkg.com/vue-lazyload/-/vue-lazyload-1.2.2.tgz#73335ed32db25264f5957df1a21d277823423743"