Merge branch '161-i18n-vue-gettext' into 'develop'

Resolve "Industrialize the i18n workflow"

Closes #161 and #167

See merge request funkwhale/funkwhale!286

.gitignore

@@ -91,3 +91,4 @@ data/
 po/*.po
 docs/swagger
 _build
+front/src/translations.json

.gitlab-ci.yml

@@ -19,9 +19,12 @@ review_front:
   when: manual
   allow_failure: true
   before_script:
+    - apt-get update
+    - apt-get install jq -y
     - cd front
   script:
     - yarn install
+    - yarn run i18n-compile
     # this is to ensure we don't have any errors in the output,
     # cf https://code.eliotberriot.com/funkwhale/funkwhale/issues/169
     - INSTANCE_URL=$REVIEW_INSTANCE_URL yarn run build | tee /dev/stderr | (! grep -i 'ERROR in')
@@ -175,11 +178,11 @@ build_front:
   stage: build
   image: node:9
   before_script:
+    - apt-get update
+    - apt-get install jq -y
     - cd front
-
   script:
     - yarn install
-    - yarn run i18n-extract
     - yarn run i18n-compile
     # this is to ensure we don't have any errors in the output,
     # cf https://code.eliotberriot.com/funkwhale/funkwhale/issues/169

CONTRIBUTING

@@ -289,8 +289,9 @@ Typical workflow for a contribution
 Internationalization
 --------------------
 
+We're using https://github.com/Polyconseil/vue-gettext to manage i18n in the project.
 When working on the front-end, any end-user string should be translated
-using either ``<i18next path="yourstring">`` or the ``$t('yourstring')``
+using either ``<translate>yourstring</translate>`` or ``$gettext('yourstring')``
 function.
 
 Extraction is done by calling ``yarn run i18n-extract``, which

README.rst

@@ -26,4 +26,9 @@ Contribute
 ----------
 
 Contribution guidelines as well as development installation instructions
-are outlined in `CONTRIBUTING <CONTRIBUTING>`_
+are outlined in `CONTRIBUTING <CONTRIBUTING>`_.
+
+Translate
+^^^^^^^^^
+
+Translators willing to help can refer to `TRANSLATORS <TRANSLATORS>`_ for instructions.

TRANSLATORS.rst

+Translating Funkwhale
+=====================
+
+Thank you for reading this! If you want to help translate Funkwhale,
+you found the proper place :)
+
+Translation is done via our own Weblate instance at https://translate.funkwhale.audio/projects/funkwhale/front/.
+
+You can signup/login using your Gitlab account (from https://code.eliotberriot.com).
+
+Translation workflow
+--------------------
+
+Once you're logged-in on the Weblate instance, you can suggest translations. Your suggestions will then be reviewer
+by the project maintainer or other translators to ensure consistency.
+
+Guidelines
+----------
+
+Respecting those guidelines is mandatory if you want your translation to be included:
+
+- Use gender-neutral language and wording
+
+Requesting a new language
+-------------------------
+
+If you'd like to see a new language in Funkwhale, please open an issue here:
+https://code.eliotberriot.com/funkwhale/funkwhale/issues

changes/changelog.d/161.feature

+New translation workflow (#161, #167)

docs/index.rst

@@ -23,6 +23,7 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in
    api
    third-party
    contributing
+   translators
    changelog
 
 Indices and tables

docs/translators.rst

+.. include:: ../TRANSLATORS.rst

front/build/dev-server.js

@@ -14,8 +14,6 @@ var webpackConfig = process.env.NODE_ENV === 'testing'
   ? require('./webpack.prod.conf')
   : require('./webpack.dev.conf')
 
-require('./i18n')
-
 // default port where dev server listens for incoming traffic
 var port = process.env.PORT || config.dev.port
 var host = process.env.HOST || config.dev.host

front/build/i18n.js

-const fs = require('fs');
-const path = require('path');
-const { gettextToI18next } = require('i18next-conv');
-
-const poDir = path.join(__dirname, '..', '..', 'po')
-const outDir = path.join(__dirname, '..', 'static', 'translations')
-if (!fs.existsSync(outDir) || !fs.statSync(outDir).isDirectory()) {
-    fs.mkdirSync(outDir)
-}
-
-// Convert .po files to i18next files
-fs.readdir(poDir, (err, files) => {
-    if (err) {
-        return console.log(err)
-    }
-
-    for (const file of files) {
-        if (file.endsWith('.po')) {
-            const lang = file.replace(/\.po$/, '')
-            const output = path.join(outDir, `${lang}.json`)
-            fs.readFile(path.join(poDir, file), (err, content) => {
-                if (err) {
-                    return console.log(err)
-                }
-
-                gettextToI18next(lang, content).then(res => {
-                    fs.writeFile(output, res, err => {
-                        if (err) {
-                            console.log(err)
-                        } else {
-                            console.log(`Wrote translation file: ${output}`)
-                            if (lang === 'en') {
-                                // for english, we need to specify that json values are equal to the keys.
-                                // otherwise we end up with empty strings on the front end for english
-                                var contents = fs.readFileSync(output)
-                                var jsonContent = JSON.parse(contents)
-                                var finalContent = {}
-                                Object.keys(jsonContent).forEach(function(key) {
-                                    finalContent[key] = key
-                                })
-                                fs.writeFile(output, JSON.stringify(finalContent))
-                            }
-                        }
-                    })
-                })
-            })
-        }
-    }
-})

front/package.json

@@ -5,11 +5,11 @@
   "author": "Eliot Berriot <contact@eliotberriot.com>",
   "private": true,
   "scripts": {
-    "dev": "node build/dev-server.js",
-    "start": "node build/dev-server.js",
+    "dev": "scripts/i18n-compile.sh && node build/dev-server.js",
+    "start": "scripts/i18n-compile.sh && node build/dev-server.js",
     "build": "node build/build.js",
-    "i18n-extract": "find src/ -name '*.vue' | xargs vendor/vue-i18n-xgettext/index.js > ../po/en.po",
-    "i18n-compile": "node build/i18n.js",
+    "i18n-extract": "scripts/i18n-extract.sh",
+    "i18n-compile": "scripts/i18n-compile.sh",
     "unit": "cross-env BABEL_ENV=test karma start test/unit/karma.conf.js --single-run",
     "unit-watch": "cross-env BABEL_ENV=test karma start test/unit/karma.conf.js",
     "e2e": "node test/e2e/runner.js",
@@ -21,9 +21,6 @@
     "axios": "^0.17.1",
     "dateformat": "^2.0.0",
     "django-channels": "^1.1.6",
-    "i18next": "^11.1.1",
-    "i18next-conv": "^6.0.0",
-    "i18next-fetch-backend": "^0.1.0",
     "js-logger": "^1.3.0",
     "jwt-decode": "^2.2.0",
     "lodash": "^4.17.4",
@@ -34,6 +31,7 @@
     "semantic-ui-css": "^2.2.10",
     "showdown": "^1.8.6",
     "vue": "^2.5.16",
+    "vue-gettext": "^2.0.31",
     "vue-lazyload": "^1.1.4",
     "vue-masonry": "^0.10.16",
     "vue-router": "^2.3.1",
@@ -61,6 +59,7 @@
     "cross-env": "^4.0.0",
     "cross-spawn": "^5.0.1",
     "css-loader": "^0.28.0",
+    "easygettext": "^2.5.0",
     "es6-promise": "^4.2.2",
     "eslint": "^3.19.0",
     "eslint-config-standard": "^6.2.1",
@@ -104,7 +103,6 @@
     "sinon-chai": "^2.8.0",
     "sinon-stub-promise": "^4.0.0",
     "url-loader": "^0.5.8",
-    "vue-i18n-xgettext": "^0.0.4",
     "vue-loader": "^12.1.0",
     "vue-style-loader": "^3.0.1",
     "vue-template-compiler": "^2.3.3",

front/scripts/i18n-compile.sh

+#!/bin/bash -eux
+locales=$(tail -n +2 src/locales.js | sed -e 's/export default //' | jq '.locales[].code' | xargs echo)
+find locales -name '*.po' | xargs $(yarn bin gettext-extract)/gettext-compile --output src/translations.json

front/scripts/i18n-extract.sh

+#!/bin/bash -eux
+locales=$(tail -n +2 src/locales.js | sed -e 's/export default //' | jq '.locales[].code' | xargs echo)
+locales_dir="locales"
+sources=$(find src -name '*.vue' -o -name '*.html' 2> /dev/null)
+js_sources=$(find src -name '*.vue' -o -name '*.js')
+touch $locales_dir/app.pot
+
+# Create a main .pot template, then generate .po files for each available language.
+# Extract gettext strings from templates files and create a POT dictionary template.
+$(yarn bin gettext-extract)/gettext-extract --attribute v-translate --quiet --output $locales_dir/app.pot $sources
+xgettext --language=JavaScript --keyword=npgettext:1c,2,3 \
+    --from-code=utf-8 --join-existing --no-wrap \
+    --package-name=$(node -e "console.log(require('./package.json').name);") \
+    --package-version=$(node -e "console.log(require('./package.json').version);") \
+    --output $locales_dir/app.pot $js_sources
+
+# Fix broken files path/lines in pot
+sed -e 's|#: src/|#: front/src/|' -i $locales_dir/app.pot
+
+# Generate .po files for each available language.
+echo $locales
+for lang in $locales; do \
+    po_file=$locales_dir/$lang/LC_MESSAGES/app.po; \
+    echo "msgmerge --update $po_file "; \
+    mkdir -p $(dirname $po_file); \
+    [ -f $po_file ] && msgmerge --lang=$lang --update $po_file $locales_dir/app.pot || msginit --no-translator --locale=$lang --input=$locales_dir/app.pot --output-file=$po_file; \
+    msgattrib --no-wrap --no-obsolete -o $po_file $po_file; \
+done;

front/src/App.vue

@@ -2,14 +2,14 @@
   <div id="app">
     <div class="ui main text container instance-chooser" v-if="!$store.state.instance.instanceUrl">
       <div class="ui padded segment">
-        <h1 class="ui header">{{ $t('Choose your instance') }}</h1>
+        <h1 class="ui header">{{ $gettext('Choose your instance') }}</h1>
         <form class="ui form" @submit.prevent="$store.dispatch('instance/setUrl', instanceUrl)">
-          <p>{{ $t('You need to select an instance in order to continue') }}</p>
+          <p>{{ $gettext('You need to select an instance in order to continue') }}</p>
           <div class="ui action input">
             <input type="text" v-model="instanceUrl">
-            <button type="submit" class="ui button">{{ $t('Submit') }}</button>
+            <button type="submit" class="ui button">{{ $gettext('Submit') }}</button>
           </div>
-          <p>{{ $t('Suggested choices') }}</p>
+          <p>{{ $gettext('Suggested choices') }}</p>
           <div class="ui bulleted list">
             <div class="ui item" v-for="url in suggestedInstances">
               <a @click="instanceUrl = url">{{ url }}</a>
@@ -27,20 +27,20 @@
         <div class="ui container">
           <div class="ui stackable equal height stackable grid">
             <div class="three wide column">
-              <i18next tag="h4" class="ui header" path="Links"></i18next>
+              <h4 v-translate class="ui header">Links</h4>
               <div class="ui link list">
                 <router-link class="item" to="/about">
-                  <i18next path="About this instance" />
+                  {{ $gettext('About this instance') }}
                 </router-link>
-                <a href="https://funkwhale.audio" class="item" target="_blank">{{ $t('Official website') }}</a>
-                <a href="https://docs.funkwhale.audio" class="item" target="_blank">{{ $t('Documentation') }}</a>
+                <a href="https://funkwhale.audio" class="item" target="_blank">{{ $gettext('Official website') }}</a>
+                <a href="https://docs.funkwhale.audio" class="item" target="_blank">{{ $gettext('Documentation') }}</a>
                 <a href="https://code.eliotberriot.com/funkwhale/funkwhale" class="item" target="_blank">
-                  <template v-if="version">{{ $t('Source code ({% version %})', {version: version}) }}</template>
-                  <template v-else>{{ $t('Source code') }}</template>
+                  <translate :translate-params="{version: version}" v-if="version">Source code (%{version})</translate>
+                  <translate v-else>Source code</translate>
                 </a>
-                <a href="https://code.eliotberriot.com/funkwhale/funkwhale/issues" class="item" target="_blank">{{ $t('Issue tracker') }}</a>
+                <a href="https://code.eliotberriot.com/funkwhale/funkwhale/issues" class="item" target="_blank">{{ $gettext('Issue tracker') }}</a>
                 <a @click="switchInstance" class="item" >
-                  {{ $t('Use another instance') }}
+                  {{ $gettext('Use another instance') }}
                   <template v-if="$store.state.instance.instanceUrl !== '/'">
                     <br>
                     ({{ $store.state.instance.instanceUrl }})
@@ -49,14 +49,26 @@
               </div>
             </div>
             <div class="ten wide column">
-              <i18next tag="h4" class="ui header" path="About funkwhale" />
+              <h4 v-translate class="ui header">About Funkwhale</h4>
               <p>
-                <i18next path="Funkwhale is a free and open-source project run by volunteers. You can help us improve the platform by reporting bugs, suggesting features and share the project with your friends!"/>
+                <translate>Funkwhale is a free and open-source project run by volunteers. You can help us improve the platform by reporting bugs, suggesting features and share the project with your friends!</translate>
               </p>
               <p>
-                <i18next path="The funkwhale logo was kindly designed and provided by Francis Gading."/>
+                <translate>The funkwhale logo was kindly designed and provided by Francis Gading.</translate>
               </p>
             </div>
+            <div class="three wide column">
+              <h4 v-translate class="ui header">Options</h4>
+              <div class="ui form">
+                <div class="ui field">
+                  <label>{{ $gettext('Change language') }}</label>
+                  <select class="ui dropdown" v-model="$language.current">
+                    <option v-for="(language, key) in $language.available" :value="key">{{ language }}</option>
+                  </select>
+                </div>
+              </div>
+            </div>
+
           </div>
         </div>
       </div>
@@ -115,7 +127,7 @@ export default {
       })
     },
     switchInstance () {
-      let confirm = window.confirm(this.$t('This will erase your local data and disconnect you, do you want to continue?'))
+      let confirm = window.confirm(this.$gettext('This will erase your local data and disconnect you, do you want to continue?'))
       if (confirm) {
         this.$store.commit('instance/instanceUrl', null)
       }
@@ -144,6 +156,9 @@ export default {
     '$store.state.instance.instanceUrl' () {
       this.$store.dispatch('instance/fetchSettings')
       this.fetchNodeInfo()
+    },
+    '$language.current' (newValue) {
+      this.$store.commit('ui/currentLanguage', newValue)
     }
   }
 }

front/src/components/About.vue

@@ -3,21 +3,23 @@
     <div class="ui vertical center aligned stripe segment">
       <div class="ui text container">
         <h1 class="ui huge header">
-            <template v-if="instance.name.value">{{ $t('About {%instance%}', { instance: instance.name.value }) }}</template>
-            <template v-else="instance.name.value">{{ $t('About this instance') }}</template>
+            <template v-if="instance.name.value" :template-params="{instance: instance.name}">
+             About %{ instance }
+            </template>
+            <template v-else="instance.name.value">{{ $gettext('About this instance') }}</template>
         </h1>
         <stats></stats>
       </div>
     </div>
     <div class="ui vertical stripe segment">
       <p v-if="!instance.short_description.value && !instance.long_description.value">
-        {{ $t('Unfortunately, owners of this instance did not yet take the time to complete this page.') }}
+        {{ $gettext('Unfortunately, owners of this instance did not yet take the time to complete this page.') }}
       </p>
       <router-link
         class="ui button"
         v-if="$store.state.auth.availablePermissions['settings']"
         :to="{path: '/manage/settings', hash: 'instance'}">
-        <i class="pencil icon"></i>{{ $t('Edit instance info') }}
+        <i class="pencil icon"></i>{{ $gettext('Edit instance info') }}
       </router-link>
       <div
         v-if="instance.short_description.value"

front/src/components/Home.vue

@@ -3,15 +3,15 @@
     <div class="ui vertical center aligned stripe segment">
       <div class="ui text container">
         <h1 class="ui huge header">
-          {{ $t('Welcome on Funkwhale') }}
+          {{ $gettext('Welcome on Funkwhale') }}
         </h1>
-        <p>{{ $t('We think listening to music should be simple.') }}</p>
+        <p>{{ $gettext('We think listening to music should be simple.') }}</p>
         <router-link class="ui icon button" to="/about">
           <i class="info icon"></i>
-          {{ $t('Learn more about this instance') }}
+          {{ $gettext('Learn more about this instance') }}
         </router-link>
         <router-link class="ui icon teal button" to="/library">
-          {{ $t('Get me to the library') }}
+          {{ $gettext('Get me to the library') }}
           <i class="right arrow icon"></i>
         </router-link>
       </div>
@@ -22,9 +22,9 @@
           <div class="row">
             <div class="eight wide left floated column">
               <h2 class="ui header">
-                {{ $t('Why funkwhale?') }}
+                {{ $gettext('Why funkwhale?') }}
               </h2>
-              <p>{{ $t('That\'s simple: we loved Grooveshark and we want to build something even better.') }}</p>
+              <p>{{ $gettext('That\'s simple: we loved Grooveshark and we want to build something even better.') }}</p>
             </div>
             <div class="four wide left floated column">
               <img class="ui medium image" src="../assets/logo/logo.png" />
@@ -35,26 +35,26 @@
       <div class="ui middle aligned stackable text container">
         <div class="ui hidden divider"></div>
         <h2 class="ui header">
-          {{ $t('Unlimited music') }}
+          {{ $gettext('Unlimited music') }}
         </h2>
-        <p>{{ $t('Funkwhale is designed to make it easy to listen to music you like, or to discover new artists.') }}</p>
+        <p>{{ $gettext('Funkwhale is designed to make it easy to listen to music you like, or to discover new artists.') }}</p>
         <div class="ui list">
           <div class="item">
             <i class="sound icon"></i>
             <div class="content">
-              {{ $t('Click once, listen for hours using built-in radios') }}
+              {{ $gettext('Click once, listen for hours using built-in radios') }}
             </div>
           </div>
           <div class="item">
             <i class="heart icon"></i>
             <div class="content">
-              {{ $t('Keep a track of your favorite songs') }}
+              {{ $gettext('Keep a track of your favorite songs') }}
             </div>
           </div>
           <div class="item">
             <i class="list icon"></i>
             <div class="content">
-              {{ $t('Playlists? We got them') }}
+              {{ $gettext('Playlists? We got them') }}
             </div>
           </div>
         </div>
@@ -62,28 +62,31 @@
       <div class="ui middle aligned stackable text container">
         <div class="ui hidden divider"></div>
         <h2 class="ui header">
-          {{ $t('Clean library') }}
+          {{ $gettext('Clean library') }}
         </h2>
-        <p>{{ $t('Funkwhale takes care of handling your music') }}.</p>
+        <p>{{ $gettext('Funkwhale takes care of handling your music') }}.</p>
         <div class="ui list">
           <div class="item">
             <i class="download icon"></i>
             <div class="content">
-              {{ $t('Import music from various platforms, such as YouTube or SoundCloud') }}
+              {{ $gettext('Import music from various platforms, such as YouTube or SoundCloud') }}
             </div>
           </div>
           <div class="item">
             <i class="tag icon"></i>
             <div class="content">
-              <i18next path="Get quality metadata about your music thanks to {%0%}">
-                <a href="https://musicbrainz.org" target="_blank">{{ $t('MusicBrainz') }}</a>
-              </i18next>
+              <template v-translate>
+                Get quality metadata about your music thanks to
+                <a href="https://musicbrainz.org" target="_blank">
+                  MusicBrainz
+                </a>
+              </template>
             </div>
           </div>
           <div class="item">
             <i class="plus icon"></i>
             <div class="content">
-              {{ $t('Covers, lyrics, our goal is to have them all ;)') }}
+              {{ $gettext('Covers, lyrics, our goal is to have them all ;)') }}
             </div>
           </div>
         </div>
@@ -91,20 +94,20 @@
       <div class="ui middle aligned stackable text container">
         <div class="ui hidden divider"></div>
         <h2 class="ui header">
-          {{ $t('Easy to use') }}
+          {{ $gettext('Easy to use') }}
         </h2>
-        <p>{{ $t('Funkwhale is dead simple to use.') }}</p>
+        <p>{{ $gettext('Funkwhale is dead simple to use.') }}</p>
         <div class="ui list">
           <div class="item">
             <i class="book icon"></i>
             <div class="content">
-              {{ $t('No add-ons, no plugins : you only need a web library') }}
+              {{ $gettext('No add-ons, no plugins : you only need a web library') }}
             </div>
           </div>
           <div class="item">
             <i class="wizard icon"></i>
             <div class="content">
-              {{ $t('Access your music from a clean interface that focus on what really matters') }}
+              {{ $gettext('Access your music from a clean interface that focus on what really matters') }}
             </div>
           </div>
         </div>
@@ -112,26 +115,26 @@
       <div class="ui middle aligned stackable text container">
         <div class="ui hidden divider"></div>
         <h2 class="ui header">
-          {{ $t('Your music, your way') }}
+          {{ $gettext('Your music, your way') }}
         </h2>
-        <p>{{ $t('Funkwhale is free and gives you control on your music.') }}</p>
+        <p>{{ $gettext('Funkwhale is free and gives you control on your music.') }}</p>
         <div class="ui list">
           <div class="item">
             <i class="smile icon"></i>
             <div class="content">
-              {{ $t('The plaform is free and open-source, you can install it and modify it without worries') }}
+              {{ $gettext('The plaform is free and open-source, you can install it and modify it without worries') }}
             </div>
           </div>
           <div class="item">
             <i class="protect icon"></i>
             <div class="content">
-              {{ $t('We do not track you or bother you with ads') }}
+              {{ $gettext('We do not track you or bother you with ads') }}
             </div>
           </div>
           <div class="item">
             <i class="users icon"></i>
             <div class="content">
-              {{ $t('You can invite friends and family to your instance so they can enjoy your music') }}
+              {{ $gettext('You can invite friends and family to your instance so they can enjoy your music') }}
             </div>
           </div>
         </div>

front/src/components/PageNotFound.vue

@@ -5,13 +5,14 @@
         <h1 class="ui huge header">
           <i class="warning icon"></i>
           <div class="content">
-            <strike>{{ $t('Whale') }}</strike> {{ $t('Page not found!') }}
+            <strike>{{ $gettext('Whale') }}</strike> {{ $gettext('Page not found!') }}
           </div>
         </h1>
-        <p>{{ $t('We\'re sorry, the page you asked for does not exists.') }}</p>
-        <i18next path="Requested URL: {%0%}"><a :href="path">{{ path }}</a></i18next>
+        <p>{{ $gettext('We\'re sorry, the page you asked for does not exist:') }}</p>
+        <a :href="path">{{ path }}</a>
+        <div class="ui hidden divider"></div>
         <router-link class="ui icon button" to="/">
-          {{ $t('Go to home page') }}
+          {{ $gettext('Go to home page') }}
           <i class="right arrow icon"></i>
         </router-link>
       </div>