Use Intl for names of languages (#11427)

* Use Intl for names of languages

* Tweak Intl language style from "American English" -> "US English"

* Update tests

* Fix tests

* Consolidate languageHandler-test files

* Improve coverage

* Consistent casing for languages in dropdown

* Update LanguageDropdown.tsx

* Delint & update snapshot

* Fix tests

* Improve coverage

`of` will fallback to the given code with fallback=code (default)

__mocks__/languages.json

@@ -1,10 +1,4 @@
 {
-    "en": {
+    "en": "en_EN.json",
-        "fileName": "en_EN.json",
+    "en-us": "en_US.json"
-        "label": "English"
-    },
-    "en-us": {
-        "fileName": "en_US.json",
-        "label": "English (US)"
-    }
 }

cypress/e2e/settings/general-user-settings-tab.spec.ts

@@ -133,10 +133,12 @@ describe("General user settings tab", () => {
                     cy.findByRole("button", { name: "Language Dropdown" }).click();
 
                     // Assert that the default option is rendered and highlighted
-                    cy.findByRole("option", { name: /Bahasa Indonesia/ })
+                    cy.findByRole("option", { name: /Albanian/ })
                         .should("be.visible")
                         .should("have.class", "mx_Dropdown_option_highlight");
 
+                    cy.findByRole("option", { name: /Deutsch/ }).should("be.visible");
+
                     // Click again to close the dropdown
                     cy.findByRole("button", { name: "Language Dropdown" }).click();
 

res/css/_components.pcss

@@ -190,6 +190,7 @@
 @import "./views/elements/_InteractiveTooltip.pcss";
 @import "./views/elements/_InviteReason.pcss";
 @import "./views/elements/_LabelledCheckbox.pcss";
+@import "./views/elements/_LanguageDropdown.pcss";
 @import "./views/elements/_MiniAvatarUploader.pcss";
 @import "./views/elements/_Pill.pcss";
 @import "./views/elements/_PowerSelector.pcss";

res/css/views/elements/_LanguageDropdown.pcss

@@ -0,0 +1,21 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_LanguageDropdown {
+    .mx_Dropdown_option > div {
+        text-transform: capitalize;
+    }
+}

src/components/views/auth/CountryDropdown.tsx

@@ -70,7 +70,7 @@ export default class CountryDropdown extends React.Component<IProps, IState> {
                 const locale = new Intl.Locale(navigator.language ?? navigator.languages[0]);
                 const code = locale.region ?? locale.language ?? locale.baseName;
                 const displayNames = new Intl.DisplayNames(["en"], { type: "region" });
-                const displayName = displayNames.of(code)?.toUpperCase();
+                const displayName = displayNames.of(code)!.toUpperCase();
                 defaultCountry = COUNTRIES.find(
                     (c) => c.iso2 === code.toUpperCase() || c.name.toUpperCase() === displayName,
                 );

src/components/views/elements/LanguageDropdown.tsx

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import React, { ReactElement } from "react";
+import classNames from "classnames";
 
 import * as languageHandler from "../../../languageHandler";
 import SettingsStore from "../../../settings/SettingsStore";
@@ -24,9 +25,10 @@ import Spinner from "./Spinner";
 import Dropdown from "./Dropdown";
 import { NonEmptyArray } from "../../../@types/common";
 
-type Languages = Awaited<ReturnType<typeof languageHandler.getAllLanguagesFromJson>>;
+type Languages = Awaited<ReturnType<typeof languageHandler.getAllLanguagesWithLabels>>;
 
 function languageMatchesSearchQuery(query: string, language: Languages[0]): boolean {
+    if (language.labelInTargetLanguage.toUpperCase().includes(query.toUpperCase())) return true;
     if (language.label.toUpperCase().includes(query.toUpperCase())) return true;
     if (language.value.toUpperCase() === query.toUpperCase()) return true;
     return false;
@@ -56,23 +58,30 @@ export default class LanguageDropdown extends React.Component<IProps, IState> {
 
     public componentDidMount(): void {
         languageHandler
-            .getAllLanguagesFromJson()
+            .getAllLanguagesWithLabels()
             .then((langs) => {
                 langs.sort(function (a, b) {
-                    if (a.label < b.label) return -1;
+                    if (a.labelInTargetLanguage < b.labelInTargetLanguage) return -1;
-                    if (a.label > b.label) return 1;
+                    if (a.labelInTargetLanguage > b.labelInTargetLanguage) return 1;
                     return 0;
                 });
                 this.setState({ langs });
             })
             .catch(() => {
-                this.setState({ langs: [{ value: "en", label: "English" }] });
+                this.setState({
+                    langs: [
+                        {
+                            value: "en",
+                            label: "English",
+                            labelInTargetLanguage: "English",
+                        },
+                    ],
+                });
             });
 
         if (!this.props.value) {
-            // If no value is given, we start with the first
+            // If no value is given, we start with the first country selected,
-            // country selected, but our parent component
+            // but our parent component doesn't know this, therefore we do this.
-            // doesn't know this, therefore we do this.
             const language = languageHandler.getUserLanguage();
             this.props.onOptionChange(language);
         }
@@ -89,7 +98,7 @@ export default class LanguageDropdown extends React.Component<IProps, IState> {
             return <Spinner />;
         }
 
-        let displayedLanguages: Awaited<ReturnType<typeof languageHandler.getAllLanguagesFromJson>>;
+        let displayedLanguages: Awaited<ReturnType<typeof languageHandler.getAllLanguagesWithLabels>>;
         if (this.state.searchQuery) {
             displayedLanguages = this.state.langs.filter((lang) => {
                 return languageMatchesSearchQuery(this.state.searchQuery, lang);
@@ -99,7 +108,7 @@ export default class LanguageDropdown extends React.Component<IProps, IState> {
         }
 
         const options = displayedLanguages.map((language) => {
-            return <div key={language.value}>{language.label}</div>;
+            return <div key={language.value}>{language.labelInTargetLanguage}</div>;
         }) as NonEmptyArray<ReactElement & { key: string }>;
 
         // default value here too, otherwise we need to handle null / undefined
@@ -116,7 +125,7 @@ export default class LanguageDropdown extends React.Component<IProps, IState> {
         return (
             <Dropdown
                 id="mx_LanguageDropdown"
-                className={this.props.className}
+                className={classNames("mx_LanguageDropdown", this.props.className)}
                 onOptionChange={this.props.onOptionChange}
                 onSearchChange={this.onSearchChange}
                 searchEnabled={true}

src/components/views/elements/SpellCheckLanguagesDropdown.tsx

@@ -19,12 +19,14 @@ import React, { ReactElement } from "react";
 import Dropdown from "../../views/elements/Dropdown";
 import PlatformPeg from "../../../PlatformPeg";
 import SettingsStore from "../../../settings/SettingsStore";
-import { _t } from "../../../languageHandler";
+import { _t, getUserLanguage } from "../../../languageHandler";
 import Spinner from "./Spinner";
-import * as languageHandler from "../../../languageHandler";
 import { NonEmptyArray } from "../../../@types/common";
 
-type Languages = Awaited<ReturnType<typeof languageHandler.getAllLanguagesFromJson>>;
+type Languages = {
+    value: string;
+    label: string; // translated
+}[];
 function languageMatchesSearchQuery(query: string, language: Languages[0]): boolean {
     if (language.label.toUpperCase().includes(query.toUpperCase())) return true;
     if (language.value.toUpperCase() === query.toUpperCase()) return true;
@@ -58,6 +60,7 @@ export default class SpellCheckLanguagesDropdown extends React.Component<
     public componentDidMount(): void {
         const plaf = PlatformPeg.get();
         if (plaf) {
+            const languageNames = new Intl.DisplayNames([getUserLanguage()], { type: "language", style: "short" });
             plaf.getAvailableSpellCheckLanguages()
                 ?.then((languages) => {
                     languages.sort(function (a, b) {
@@ -68,7 +71,7 @@ export default class SpellCheckLanguagesDropdown extends React.Component<
                     const langs: Languages = [];
                     languages.forEach((language) => {
                         langs.push({
-                            label: language,
+                            label: languageNames.of(language)!,
                             value: language,
                         });
                     });
@@ -79,7 +82,7 @@ export default class SpellCheckLanguagesDropdown extends React.Component<
                         languages: [
                             {
                                 value: "en",
-                                label: "English",
+                                label: languageNames.of("en")!,
                             },
                         ],
                     });

src/languageHandler.tsx

@@ -433,10 +433,7 @@ export function setMissingEntryGenerator(f: (value: string) => void): void {
 }
 
 type Languages = {
-    [lang: string]: {
+    [lang: string]: string;
-        fileName: string;
-        label: string;
-    };
 };
 
 export function setLanguage(preferredLangs: string | string[]): Promise<void> {
@@ -467,7 +464,7 @@ export function setLanguage(preferredLangs: string | string[]): Promise<void> {
                 logger.error("Unable to find an appropriate language");
             }
 
-            return getLanguageRetry(i18nFolder + availLangs[langToUse].fileName);
+            return getLanguageRetry(i18nFolder + availLangs[langToUse]);
         })
         .then(async (langData): Promise<ICounterpartTranslation | undefined> => {
             counterpart.registerTranslations(langToUse, langData);
@@ -481,7 +478,7 @@ export function setLanguage(preferredLangs: string | string[]): Promise<void> {
 
             // Set 'en' as fallback language:
             if (langToUse !== "en") {
-                return getLanguageRetry(i18nFolder + availLangs["en"].fileName);
+                return getLanguageRetry(i18nFolder + availLangs["en"]);
             }
         })
         .then(async (langData): Promise<void> => {
@@ -492,21 +489,23 @@ export function setLanguage(preferredLangs: string | string[]): Promise<void> {
 
 type Language = {
     value: string;
-    label: string;
+    label: string; // translated
+    labelInTargetLanguage: string; // translated
 };
 
-export function getAllLanguagesFromJson(): Promise<Language[]> {
+export async function getAllLanguagesFromJson(): Promise<string[]> {
-    return getLangsJson().then((langsObject) => {
+    return Object.keys(await getLangsJson());
-        const langs: Language[] = [];
+}
-        for (const langKey in langsObject) {
+
-            if (langsObject.hasOwnProperty(langKey)) {
+export async function getAllLanguagesWithLabels(): Promise<Language[]> {
-                langs.push({
+    const languageNames = new Intl.DisplayNames([getUserLanguage()], { type: "language", style: "short" });
+    const languages = await getAllLanguagesFromJson();
+    return languages.map<Language>((langKey) => {
+        return {
             value: langKey,
-                    label: langsObject[langKey].label,
+            label: languageNames.of(langKey)!,
-                });
+            labelInTargetLanguage: new Intl.DisplayNames([langKey], { type: "language", style: "short" }).of(langKey)!,
-            }
+        };
-        }
-        return langs;
     });
 }
 

test/components/structures/__snapshots__/MatrixChat-test.tsx.snap

@@ -37,7 +37,7 @@ exports[`<MatrixChat /> with a soft-logged-out session should show the soft-logo
           Matrix
         </aside>
         <div
-          class="mx_Dropdown mx_AuthBody_language"
+          class="mx_Dropdown mx_LanguageDropdown mx_AuthBody_language"
         >
           <div
             aria-describedby="mx_LanguageDropdown_value"

test/components/views/elements/SpellCheckLanguagesDropdown-test.tsx

@@ -0,0 +1,38 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import { render, screen, waitForElementToBeRemoved } from "@testing-library/react";
+
+import SpellCheckLanguagesDropdown from "../../../../src/components/views/elements/SpellCheckLanguagesDropdown";
+import PlatformPeg from "../../../../src/PlatformPeg";
+
+describe("<SpellCheckLanguagesDropdown />", () => {
+    it("renders as expected", async () => {
+        const platform: any = { getAvailableSpellCheckLanguages: jest.fn().mockResolvedValue(["en", "de", "qq"]) };
+        PlatformPeg.set(platform);
+
+        const { asFragment } = render(
+            <SpellCheckLanguagesDropdown
+                className="mx_GeneralUserSettingsTab_spellCheckLanguageInput"
+                value="en"
+                onOptionChange={jest.fn()}
+            />,
+        );
+        await waitForElementToBeRemoved(() => screen.queryAllByLabelText("Loading…"));
+        expect(asFragment()).toMatchSnapshot();
+    });
+});

test/components/views/elements/__snapshots__/SpellCheckLanguagesDropdown-test.tsx.snap

@@ -0,0 +1,32 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`<SpellCheckLanguagesDropdown /> renders as expected 1`] = `
+<DocumentFragment>
+  <div
+    class="mx_Dropdown mx_GeneralUserSettingsTab_spellCheckLanguageInput"
+  >
+    <div
+      aria-describedby="mx_LanguageDropdown_value"
+      aria-expanded="false"
+      aria-haspopup="listbox"
+      aria-label="Language Dropdown"
+      aria-owns="mx_LanguageDropdown_input"
+      class="mx_AccessibleButton mx_Dropdown_input mx_no_textinput"
+      role="button"
+      tabindex="0"
+    >
+      <div
+        class="mx_Dropdown_option"
+        id="mx_LanguageDropdown_value"
+      >
+        <div>
+          English
+        </div>
+      </div>
+      <span
+        class="mx_Dropdown_arrow"
+      />
+    </div>
+  </div>
+</DocumentFragment>
+`;

test/i18n/languages.json

@@ -1,6 +1,3 @@
 {
-    "en": {
+    "en": "en_EN.json"
-        "fileName": "en_EN.json",
-        "label": "English"
-    }
 }

test/languageHandler-test.ts

@@ -1,121 +0,0 @@
-/*
-Copyright 2022 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import SdkConfig from "../src/SdkConfig";
-import {
-    _t,
-    CustomTranslationOptions,
-    ICustomTranslations,
-    registerCustomTranslations,
-    setLanguage,
-    UserFriendlyError,
-} from "../src/languageHandler";
-
-async function setupTranslationOverridesForTests(overrides: ICustomTranslations) {
-    const lookupUrl = "/translations.json";
-    const fn = (url: string): ICustomTranslations => {
-        expect(url).toEqual(lookupUrl);
-        return overrides;
-    };
-
-    SdkConfig.add({
-        custom_translations_url: lookupUrl,
-    });
-    CustomTranslationOptions.lookupFn = fn;
-    await registerCustomTranslations({
-        testOnlyIgnoreCustomTranslationsCache: true,
-    });
-}
-
-describe("languageHandler", () => {
-    afterEach(() => {
-        SdkConfig.reset();
-        CustomTranslationOptions.lookupFn = undefined;
-    });
-
-    it("should support overriding translations", async () => {
-        const str = "This is a test string that does not exist in the app.";
-        const enOverride = "This is the English version of a custom string.";
-        const deOverride = "This is the German version of a custom string.";
-
-        // First test that overrides aren't being used
-        await setLanguage("en");
-        expect(_t(str)).toEqual(str);
-        await setLanguage("de");
-        expect(_t(str)).toEqual(str);
-
-        await setupTranslationOverridesForTests({
-            [str]: {
-                en: enOverride,
-                de: deOverride,
-            },
-        });
-
-        // Now test that they *are* being used
-        await setLanguage("en");
-        expect(_t(str)).toEqual(enOverride);
-
-        await setLanguage("de");
-        expect(_t(str)).toEqual(deOverride);
-    });
-
-    describe("UserFriendlyError", () => {
-        const testErrorMessage = "This email address is already in use (%(email)s)";
-        beforeEach(async () => {
-            // Setup some  strings with variable substituations that we can use in the tests.
-            const deOverride = "Diese E-Mail-Adresse wird bereits verwendet (%(email)s)";
-            await setupTranslationOverridesForTests({
-                [testErrorMessage]: {
-                    en: testErrorMessage,
-                    de: deOverride,
-                },
-            });
-        });
-
-        it("includes English message and localized translated message", async () => {
-            await setLanguage("de");
-
-            const friendlyError = new UserFriendlyError(testErrorMessage, {
-                email: "test@example.com",
-                cause: undefined,
-            });
-
-            // Ensure message is in English so it's readable in the logs
-            expect(friendlyError.message).toStrictEqual("This email address is already in use (test@example.com)");
-            // Ensure the translated message is localized appropriately
-            expect(friendlyError.translatedMessage).toStrictEqual(
-                "Diese E-Mail-Adresse wird bereits verwendet (test@example.com)",
-            );
-        });
-
-        it("includes underlying cause error", async () => {
-            await setLanguage("de");
-
-            const underlyingError = new Error("Fake underlying error");
-            const friendlyError = new UserFriendlyError(testErrorMessage, {
-                email: "test@example.com",
-                cause: underlyingError,
-            });
-
-            expect(friendlyError.cause).toStrictEqual(underlyingError);
-        });
-
-        it("ok to omit the substitution variables and cause object, there just won't be any cause", async () => {
-            const friendlyError = new UserFriendlyError("foo error");
-            expect(friendlyError.cause).toBeUndefined();
-        });
-    });
-});

test/languageHandler-test.tsx

@@ -15,18 +15,160 @@ limitations under the License.
 */
 
 import React from "react";
+import fetchMock from "fetch-mock-jest";
 
+import SdkConfig from "../src/SdkConfig";
 import {
     _t,
     _tDom,
-    TranslatedString,
+    CustomTranslationOptions,
+    getAllLanguagesWithLabels,
+    ICustomTranslations,
+    registerCustomTranslations,
     setLanguage,
     setMissingEntryGenerator,
     substitute,
-} from "../../src/languageHandler";
+    TranslatedString,
-import { stubClient } from "../test-utils";
+    UserFriendlyError,
+} from "../src/languageHandler";
+import { stubClient } from "./test-utils";
+import { setupLanguageMock } from "./setup/setupLanguage";
 
-describe("languageHandler", function () {
+async function setupTranslationOverridesForTests(overrides: ICustomTranslations) {
+    const lookupUrl = "/translations.json";
+    const fn = (url: string): ICustomTranslations => {
+        expect(url).toEqual(lookupUrl);
+        return overrides;
+    };
+
+    SdkConfig.add({
+        custom_translations_url: lookupUrl,
+    });
+    CustomTranslationOptions.lookupFn = fn;
+    await registerCustomTranslations({
+        testOnlyIgnoreCustomTranslationsCache: true,
+    });
+}
+
+describe("languageHandler", () => {
+    beforeEach(async () => {
+        await setLanguage("en");
+    });
+
+    afterEach(() => {
+        SdkConfig.reset();
+        CustomTranslationOptions.lookupFn = undefined;
+    });
+
+    it("should support overriding translations", async () => {
+        const str = "This is a test string that does not exist in the app.";
+        const enOverride = "This is the English version of a custom string.";
+        const deOverride = "This is the German version of a custom string.";
+
+        // First test that overrides aren't being used
+        await setLanguage("en");
+        expect(_t(str)).toEqual(str);
+        await setLanguage("de");
+        expect(_t(str)).toEqual(str);
+
+        await setupTranslationOverridesForTests({
+            [str]: {
+                en: enOverride,
+                de: deOverride,
+            },
+        });
+
+        // Now test that they *are* being used
+        await setLanguage("en");
+        expect(_t(str)).toEqual(enOverride);
+
+        await setLanguage("de");
+        expect(_t(str)).toEqual(deOverride);
+    });
+
+    describe("UserFriendlyError", () => {
+        const testErrorMessage = "This email address is already in use (%(email)s)";
+        beforeEach(async () => {
+            // Setup some  strings with variable substituations that we can use in the tests.
+            const deOverride = "Diese E-Mail-Adresse wird bereits verwendet (%(email)s)";
+            await setupTranslationOverridesForTests({
+                [testErrorMessage]: {
+                    en: testErrorMessage,
+                    de: deOverride,
+                },
+            });
+        });
+
+        it("includes English message and localized translated message", async () => {
+            await setLanguage("de");
+
+            const friendlyError = new UserFriendlyError(testErrorMessage, {
+                email: "test@example.com",
+                cause: undefined,
+            });
+
+            // Ensure message is in English so it's readable in the logs
+            expect(friendlyError.message).toStrictEqual("This email address is already in use (test@example.com)");
+            // Ensure the translated message is localized appropriately
+            expect(friendlyError.translatedMessage).toStrictEqual(
+                "Diese E-Mail-Adresse wird bereits verwendet (test@example.com)",
+            );
+        });
+
+        it("includes underlying cause error", async () => {
+            await setLanguage("de");
+
+            const underlyingError = new Error("Fake underlying error");
+            const friendlyError = new UserFriendlyError(testErrorMessage, {
+                email: "test@example.com",
+                cause: underlyingError,
+            });
+
+            expect(friendlyError.cause).toStrictEqual(underlyingError);
+        });
+
+        it("ok to omit the substitution variables and cause object, there just won't be any cause", async () => {
+            const friendlyError = new UserFriendlyError("foo error");
+            expect(friendlyError.cause).toBeUndefined();
+        });
+    });
+
+    describe("getAllLanguagesWithLabels", () => {
+        it("should handle unknown language sanely", async () => {
+            fetchMock.getOnce(
+                "/i18n/languages.json",
+                {
+                    en: "en_EN.json",
+                    de: "de_DE.json",
+                    qq: "qq.json",
+                },
+                { overwriteRoutes: true },
+            );
+            await expect(getAllLanguagesWithLabels()).resolves.toMatchInlineSnapshot(`
+                [
+                  {
+                    "label": "English",
+                    "labelInTargetLanguage": "English",
+                    "value": "en",
+                  },
+                  {
+                    "label": "German",
+                    "labelInTargetLanguage": "Deutsch",
+                    "value": "de",
+                  },
+                  {
+                    "label": "qq",
+                    "labelInTargetLanguage": "qq",
+                    "value": "qq",
+                  },
+                ]
+            `);
+            setupLanguageMock(); // restore language mock
+        });
+    });
+});
+
+describe("languageHandler JSX", function () {
     // See setupLanguage.ts for how we are stubbing out translations to provide fixture data for these tests
     const basicString = "Rooms";
     const selfClosingTagSub = "Accept <policyLink /> to continue:";

test/setup/setupLanguage.ts

@@ -32,24 +32,18 @@ const lv = {
 // de_DE.json
 // lv.json - mock version with few translations, used to test fallback translation
 
-fetchMock
+export function setupLanguageMock() {
+    fetchMock
         .get("/i18n/languages.json", {
-        en: {
+            en: "en_EN.json",
-            fileName: "en_EN.json",
+            de: "de_DE.json",
-            label: "English",
+            lv: "lv.json",
-        },
-        de: {
-            fileName: "de_DE.json",
-            label: "German",
-        },
-        lv: {
-            fileName: "lv.json",
-            label: "Latvian",
-        },
         })
         .get("end:en_EN.json", en)
         .get("end:de_DE.json", de)
         .get("end:lv.json", lv);
+}
+setupLanguageMock();
 
 languageHandler.setLanguage("en");
 languageHandler.setMissingEntryGenerator((key) => key.split("|", 2)[1]);