1 /**
2 Internationalization compatible with $(LINK2 https://www.gnu.org/software/gettext/, GNU gettext).
3 
4 Insert the following line at the top of your main function:
5 ---
6 mixin(gettext.main);
7 ---
8 
9 Translatable strings are marked by instantiating the `tr` template, like so:
10 ---
11 writeln(tr!"Translatable message");
12 ---
13 
14 A translation may require a particular plural form depending on a number. This can be
15 achieved by supplying both singular and plural forms as compile time arguments, and the
16 number as a runtime argument.
17 ---
18 writeln(tr!("one green bottle hanging on the wall",
19             "%d green bottles hanging on the wall")(n));
20 ---
21 
22 Plural forms can be used in format strings, but the argument that determines the form
23 must be supplied to `tr` and not to `format`. The corresponding format specifier will
24 not be seen by `format` as it will have been replaced with a string by `tr`:
25 ---
26 format(tr!("Welcome %s, you may make a wish",
27            "Welcome %s, you may make %d wishes")(n), name);
28 ---
29 The format specifier that selects the form is the last specifier in the format string
30 (here `%d`). In many sentences, however, the specifier that should select the form cannot
31 be the last. In these cases, format specifiers must be given a position argument, where
32 the highest position determines the form:
33 ---
34 foreach (i, where; [tr!"hand", tr!"bush"])
35     format(tr!("One bird in the %1$s", "%2$d birds in the %1$s")(i + 1), where);
36 ---
37 Again, the specifier with the highest position argument will never be seen by format.
38 
39 Two identical strings that have different meanings dependent on context may need to be
40 translated differently. Dis can be accomplished by disambiguating the string with a
41 context argument. It is also possible to attach a comment that will be seen by
42 the translator:
43 ---
44 auto message1 = tr!("Review the draft.", [Tr.context: "document"]);
45 auto message2 = tr!("Review the draft.", [Tr.context: "nautical",
46                                           Tr.note: `Nautical term! "Draft" = how deep the bottom` ~
47                                                    `of the ship is below the water level.`]);
48 ---
49 
50 If you'd rather use an underscore to mark translatable strings,
51 [as the GNU gettext documentation suggests](https://www.gnu.org/software/gettext/manual/html_node/Mark-Keywords.html),
52 you can use an alias:
53 ---
54 import gettext : _ = tr;    // Customary in GNU software.
55 writeln(_!"Translatable message");
56 ---
57 
58 Authors:
59 $(LINK2 https://github.com/veelo, Bastiaan Veelo)
60 Copyright:
61 SARC B.V., 2022
62 License:
63 $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
64 */
65 
66 module gettext;
67 
68 /// Optional attribute categories.
69 enum Tr {
70     note,   /// Pass a note to the translator.
71     context /// Disambiguate by giving a context.
72 }
73 
74 version (xgettext) // String extraction mode.
75 {
76     /** $(NEVER_DOCUMENT) */
77     bool scan()
78     {
79         import std.getopt;
80         import std.path : baseName, buildPath, setExtension;
81         import core.runtime : Runtime;
82 
83         auto args = Runtime.args;
84         potFile = buildPath("po", args[0].baseName);
85 
86         auto helpInformation = getopt(args,
87                                       "output|o", "The path for the PO template file.", &potFile);
88         if (helpInformation.helpWanted)
89         {
90             ()@trusted{
91                 defaultGetoptPrinter("Usage:\n\tdub run --config=xgettext [-- <options>]\nOptions:", helpInformation.options);
92             }();
93         }
94         else
95             writePOT(potFile.setExtension("pot"));
96         return args.length > 0; // Always true, but the compiler has no idea.
97     }
98 
99     import std.typecons : Tuple;
100     import std.array : join;
101     import std.ascii : newline;
102 
103     private enum Format {plain, c}
104     private alias Key = Tuple!(string, "singular",
105                                string, "plural",
106                                Format, "format",
107                                string, "context");
108     private string[][Key] translatableStrings;
109     private string[][Key] comments;
110 
111     private string potFile;
112 
113     private void writePOT(string potFile) @safe
114     {
115         import std.algorithm : cmp, map, sort;
116         import std.file : mkdirRecurse, write;
117         import std.path : baseName, dirName;
118         import std.stdio;
119 
120         string header() @safe
121         {
122             import std.exception : ifThrown;
123             import std.array : join;
124             import std.json, std.process;
125             import std.string : strip;
126 
127             string rootPackage = potFile.baseName;
128 
129             JSONValue json;
130             auto piped = pipeProcess(["dub", "describe"], Redirect.stdout);
131             scope (exit) piped.pid.wait;
132             json = ()@trusted{ return piped.stdout.byLine.join.parseJSON; }();
133             rootPackage = json["rootPackage"].str.ifThrown!JSONException(potFile.baseName);
134             foreach (_package; json["packages"].arrayNoRef)
135                 if (_package["name"].str == rootPackage)
136                 {
137                     json = _package;
138                     break;
139                 }
140 
141             string thisYear()
142             {
143                 return __DATE__[$-4 .. $];
144             }
145             string title()
146             {
147                 return "# PO Template for " ~ rootPackage ~ ".";
148             }
149             string copyright()
150             {
151                 return ("# " ~ json["copyright"].str)
152                     .ifThrown!JSONException("# Copyright © YEAR THE PACKAGE'S COPYRIGHT HOLDER");
153             }
154             string license()
155             {
156                 return ("# This file is distributed under the " ~ json["license"].str ~ " license.")
157                     .ifThrown!JSONException("# This file is distributed under the same license as the " ~ rootPackage ~ " package.");
158             }
159             string author()
160             {
161                 return (){ return json["authors"].arrayNoRef.map!(a => "# " ~ a.str ~ ", " ~ thisYear ~ ".").join(newline); }()
162                     .ifThrown!JSONException("# FIRST AUTHOR <EMAIL@ADDRESS>, " ~ thisYear ~ ".");
163             }
164             string idVersion()
165             {
166                 auto gitResult = execute(["git", "describe"]);
167                 auto _version = gitResult.status == 0 ? gitResult.output.strip : "PACKAGE VERSION";
168                 return (`"Project-Id-Version: ` ~ _version ~ `\n"`);
169             }
170             string bugs()
171             {
172                 return `"Report-Msgid-Bugs-To: \n"`;
173             }
174             string creationDate()
175             {
176                 import std.datetime;
177                 return `"POT-Creation-Date: ` ~ Clock.currTime.toUTC.toISOExtString() ~ `\n"`;
178             }
179             return [title, copyright, license, author, `#`, `#, fuzzy`, `msgid ""`, `msgstr ""`,
180                     idVersion, bugs, creationDate,
181                     `"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"`,
182                     `"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"`,
183                     `"Language-Team: LANGUAGE <LL@li.org>\n"`,
184                     `"Language: \n"`,
185                     `"MIME-Version: 1.0\n"`,
186                     `"Content-Type: text/plain; charset=UTF-8\n"`,
187                     `"Content-Transfer-Encoding: 8bit\n"`,
188                     ``, ``].join(newline);
189         }
190         mkdirRecurse(potFile.dirName);
191         ()@trusted {
192             translatableStrings.rehash;
193             comments.rehash;
194         }();
195         write(potFile, header ~ translatableStrings.keys
196               .sort!((a, b) => cmp(translatableStrings[a][0], translatableStrings[b][0]) < 0)
197               .map!(key => messageFromKey(key)).join(newline));
198         writeln(potFile ~ " generated.");
199     }
200 
201     private string stringify(string str) @safe pure
202     {
203         import std.conv : text;
204 
205         return text([str])[1 .. $-1];
206     }
207 
208     private string messageFromKey(Key key) @safe
209     {
210         string message;
211         if (auto c = key in comments)
212             foreach (comment; *c)
213                 message ~= `#. ` ~ comment ~ newline;
214         message ~= `#: ` ~ translatableStrings[key].join(" ") ~ newline;
215         if (key.format == Format.c)
216             message ~= `#, c-format` ~ newline;
217         if (key.context != null)
218             message ~= `msgctxt ` ~ key.context.stringify ~ newline;
219         if (key.plural == null)
220         {
221             message ~= `msgid ` ~ key.singular.stringify ~ newline ~
222                        `msgstr ""` ~ newline;
223         }
224         else
225         {
226             message ~= `msgid ` ~ key.singular.stringify ~ newline ~
227                        `msgid_plural ` ~ key.plural.stringify ~ newline ~
228                        `msgstr[0] ""` ~ newline ~
229                        `msgstr[1] ""` ~ newline;
230         }
231         return message;
232     }
233 
234     /** $(NEVER_DOCUMENT) */
235     template tr(string singular, string[Tr] attributes = null,
236                 int line = __LINE__, string file = __FILE__, string mod = __MODULE__, string func = __FUNCTION__, Args...)
237     {
238         alias tr = tr!(singular, null, attributes,
239                        line, file, mod, func, Args);
240     }
241 
242     /** $(NEVER_DOCUMENT) */
243     template tr(string singular, string plural, string[Tr] attributes = null,
244                 int line = __LINE__, string file = __FILE__, string mod = __MODULE__, string func = __FUNCTION__, Args...)
245     {
246         static struct StrInjector
247         {
248             static this()
249             {
250                 string reference()
251                 {
252                     import std.conv : to;
253                     import std.array : join;
254                     import std.algorithm : commonPrefix;
255                     import std.path : buildPath, pathSplitter;
256 
257                     static if (func.length == 0)
258                         return file.pathSplitter.join("/") ~ ":" ~ line.to!string;
259                     else
260                         return file.pathSplitter.join("/") ~ ":" ~ line.to!string ~ "(" ~ func[commonPrefix(func, mod).length + 1 .. $] ~ ")";
261                 }
262                 Format format()
263                 {
264                     static if (Args.length > 0 || singular.hasFormatSpecifiers || (plural && plural.hasFormatSpecifiers))
265                         return Format.c;
266                     else
267                         return Format.plain;
268                 }
269                 string context()
270                 {
271                     if (auto c = Tr.context in attributes)
272                         return *c;
273                     return null;
274                 }
275                 translatableStrings.require(Key(singular, plural, format, context)) ~= reference;
276                 if (auto c = Tr.note in attributes)
277                    comments.require(Key(singular, plural, format, context)) ~= *c;
278             }
279         }
280         static if (plural == null)
281             enum tr = TranslatableString(singular);
282         else
283             enum tr = TranslatableStringPlural(singular, plural);
284     }
285 
286     private bool hasFormatSpecifiers(string fmt) pure @safe
287     {
288         import std.format : FormatSpec;
289 
290         static void ns(const(char)[] arr) {} // the simplest output range
291         auto nullSink = &ns;
292         return FormatSpec!char(fmt).writeUpToNextSpec(nullSink);
293     }
294     unittest 
295     {
296         assert ("On %2$s I eat %3$s and walk for %1$d hours.".hasFormatSpecifiers);
297         assert ("On %%2$s I eat %%3$s and walk for %1$d hours.".hasFormatSpecifiers);
298         assert (!"On %%2$s I eat %%3$s and walk for hours.".hasFormatSpecifiers);
299     }
300 }
301 else // Translation mode.
302 {
303     /**
304     Translate `message`.
305 
306     This does *not* instantiate a new function for every marked string
307     (the signature is fabricated for the sake of documentation).
308 
309     Returns: The translation of `message` if one exists in the selected
310     language, or `message` otherwise.
311     See_Also: [selectLanguage]
312 
313     Examples:
314     ---
315     writeln(tr!"Translatable message");
316     ---
317     */
318     template tr(string singular, string[Tr] attributes = null)
319     {
320         enum tr = TranslatableString(singular, attributes);
321     }
322     /**
323     Translate a message in the correct plural form.
324 
325     This does *not* instantiate a new function for every marked string
326     (the signature is fabricated for the sake of documentation).
327 
328     The first argument should be in singular form, the second in plural
329     form. Note that the format specifier `%d` is optional.
330 
331     Returns: The translation if one exists in the selected
332     language, or the corresponding original otherwise.
333     See_Also: [selectLanguage]
334 
335     Examples:
336     ---
337     writeln(tr!("There is a goose!", "There are %d geese!")(n));
338     ---
339     */
340     template tr(string singular, string plural, string[Tr] attributes = null)
341     {
342         enum tr = TranslatableStringPlural(singular, plural);
343     }
344 }
345 import std.format : format, FormatException, FormatSpec;
346 
347 /**
348 Represents a translatable string.
349 
350 This struct can for the most part be considered an implementation detail of `gettext`.
351 A template instantiation like `tr!"Greetings"` actually results in a constructor call like
352 `TranslatableString("Greetings")` in the code. This struct is callable, so that a lookup
353 of the translation happens when the struct is evaluated.
354 
355 The only reason that this struct is public is to make declarations of static arrays of
356 translatable strings less cryptic:
357 
358 ---
359 enum RGB {red, green, blue}
360 
361 // Explicit array of translatable strings:
362 immutable TranslatableString[Color.max + 1] colors1 = [RGB.red:   tr!"Red",
363                                                        RGB.green: tr!"Green",
364                                                        RGB.blue:  tr!"Blue"];
365 // Array of translatable strings where the type is derived:
366 immutable typeof(tr!"Red")[Color.max + 1] colors2 = [RGB.red:   tr!"Red",
367                                                      RGB.green: tr!"Green",
368                                                      RGB.blue:  tr!"Blue"];
369 ---
370 */
371 /*
372 This struct+template trick allows the string to be passed as template parameter without instantiating
373 a separate function for every string. https://forum.dlang.org/post/t8pqvg$20r0$1@digitalmars.com
374 */
375 @safe struct TranslatableString
376 {
377     private enum string soh = "\x01";
378     private enum string eot = "\x04";
379     private immutable(string)[] seq;
380     this (string str, string[Tr] attributes = null) nothrow
381     {
382         if (auto context = Tr.context in attributes)
383             str = soh ~ *context ~ eot ~ str;
384         seq = [str];
385     }
386     this (string[] seq) nothrow
387     {
388         this.seq = seq.idup;
389     }
390     this (immutable(string)[] seq) nothrow
391     {
392         this.seq = seq;
393     }
394     string gettext() const
395     {
396         import std.algorithm : findSplitAfter, map, startsWith;
397         import std.array : join;
398 
399         string proxy(string message)
400         {
401              if (message.startsWith(soh))
402                 return currentLanguage.gettext(message[1 .. $]).findSplitAfter(eot)[1];
403             else
404                 return currentLanguage.gettext(message);
405         }
406 
407         return seq.map!(a => proxy(a)).join;
408     }
409     alias gettext this;
410     /** Forces evaluation as translated string.
411 
412     In a limited set of circumstances, a `TranslatableString` may forcefully need to be interpreted as a string.
413     One of these cases is a *named* enum:
414 
415     ---
416     enum E {member = tr!"translation"}
417     writeln(E.member);          // "member"
418     writeln(E.member.toString); // "translation"
419     ---
420     Contrary, anonimous enums and manifest constants do not require this treatment:
421     ---
422     enum {member = tr!"translation"}
423     writeln(member); // "translation"
424     ---
425     */
426     string toString() const // Called when a tr!"" literal or constant occurs in a writeln().
427     {
428         return gettext;
429     }
430     /// idem
431     void toString(scope void delegate(scope const(char)[]) @safe sink) const
432     {
433         sink(gettext);
434     }
435     TranslatableString opBinary(string op : "~", RHS)(RHS rhs) nothrow
436     {
437         import std.traits : Unconst;
438 
439         static if (is (RHS == TranslatableString))
440             return TranslatableString(seq ~ rhs.seq);
441         else static if (is (Unconst!RHS == TranslatableString))
442             return TranslatableString(seq ~ rhs.seq.dup);
443         // TODO Add a reserved context so that ordinary strings don't accidentally get translated.
444         else static if (is (RHS == string))
445             return TranslatableString(seq ~ rhs);
446         else static if (is (Unconst!RHS == char))
447             return TranslatableString(seq ~ [rhs].idup);
448         else
449             static assert (false, "Need implementation for " ~ RHS.stringof);
450     }
451     TranslatableString opBinaryRight(string op : "~", LHS)(LHS lhs) nothrow
452     {
453         import std.traits : Unconst;
454 
455         static if (is (LHS == TranslatableString))
456             return TranslatableString(lhs.seq ~ seq);
457         static if (is (Unconst!LHS == TranslatableString))
458             return TranslatableString(lhs.seq.dup ~ seq);
459         // TODO Add a reserved context so that ordinary strings don't accidentally get translated.
460         else static if (is (LHS == string))
461             return TranslatableString([lhs] ~ seq);
462         else static if (is (LHS == char[]))
463             return TranslatableString([lhs.idup] ~ seq);
464         else static if (is (LHS == char))
465             return TranslatableString([[lhs.idup]] ~ seq);
466         else
467             static assert (false, "Need implementation for " ~ LHS.stringof);
468     }
469 }
470 @safe struct TranslatableStringPlural
471 {
472     string str, strpl;
473     this(string str, string strpl)
474     {
475         this.str = str;
476         this.strpl = strpl;
477     }
478     string opCall(size_t number) const
479     {
480         import std.algorithm : max;
481 
482         const n = cast (int) (number > int.max ? (number % 1000000) + 1000000 : number);
483         const trans =  currentLanguage.ngettext(str, strpl, n);
484         if (countFormatSpecifiers(trans) == countFormatSpecifiers(strpl))
485         {
486             try
487                 return format(trans.disableAllButLastSpecifier, n);
488             catch(FormatException e)
489             {
490                 debug throw(e);
491                 return strpl;   // Fall back on untranslated message.
492             }
493         }
494         else
495             return trans;
496     }
497 }
498 
499 private int countFormatSpecifiers(string fmt) pure @safe
500 {
501     static void ns(const(char)[] arr) {} // the simplest output range
502     auto nullSink = &ns;
503     int count = 0;
504     auto f = FormatSpec!char(fmt);
505     while (f.writeUpToNextSpec(nullSink))
506         count++;
507     return count;
508 }
509 unittest 
510 {
511     assert ("On %2$s I eat %3$s and walk for %1$d hours.".countFormatSpecifiers == 3);
512     assert ("On %%2$s I eat %%3$s and walk for %1$d hours.".countFormatSpecifiers == 1);
513 }
514 
515 private immutable(Char)[] disableAllButLastSpecifier(Char)(const Char[] inp) @safe
516 {
517     import std.array : Appender;
518     import std.conv : to;
519     import std.exception : enforce;
520     import std.typecons : tuple;
521 
522     enum Mode {undefined, inSequence, outOfSequence}
523     Mode mode = Mode.undefined;
524 
525     Appender!(Char[]) outp;
526     outp.reserve(inp.length + 10);
527     // Traverse specs, disable all of them, note where the highest index is. Re-enable that one.
528     size_t lastSpecIndex = 0, highestSpecIndex = 0, highestSpecPos = 0, specs = 0;
529     auto f = FormatSpec!Char(inp);
530     while (f.trailing.length > 0)
531     {
532         if (f.writeUpToNextSpec(outp))
533         {
534             // Mode check.
535             if (mode == Mode.undefined)
536                 mode = f.indexStart > 0 ? Mode.outOfSequence : Mode.inSequence;
537             else
538                 enforce!FormatException( mode == Mode.inSequence && f.indexStart == 0 ||
539                                         (mode == Mode.outOfSequence && f.indexStart != lastSpecIndex),
540                         `Cannot mix specifiers with and without a position argument in "` ~ inp ~ `"`);
541             // Track the highest.
542             if (f.indexStart == 0)
543                 highestSpecPos = outp[].length + 1;
544             else
545                 if (f.indexStart > highestSpecIndex)
546                 {
547                     highestSpecIndex = f.indexStart;
548                     highestSpecPos = outp[].length + 1;
549                 }
550             // disable
551             auto curFmtSpec = inp[outp[].length - specs .. $ - f.trailing.length];
552             outp ~= '%'.to!Char ~ curFmtSpec;
553             lastSpecIndex = f.indexStart;
554             specs++;
555         }
556 
557     }
558     return mode == Mode.inSequence ?
559         (outp[][0 .. highestSpecPos] ~ outp[][highestSpecPos + 1 .. $]).idup :
560         (outp[][0 .. highestSpecPos] ~ outp[][highestSpecPos + highestSpecIndex.to!string.length + 2 .. $]).idup;
561 }
562 unittest
563 {
564     import std.exception;
565 
566     assert ("Я считаю %d яблоко.".disableAllButLastSpecifier ==
567             "Я считаю %d яблоко.");
568     assert ("Last %s, in %s, I ate %d muffins".disableAllButLastSpecifier ==
569             "Last %%s, in %%s, I ate %d muffins");
570     assert ("I ate %3$d muffins on %1$s in %2$s.".disableAllButLastSpecifier ==
571             "I ate %d muffins on %%1$s in %%2$s.");
572     assertThrown("An unpositioned %d specifier mixed with positioned specifier %3$s".disableAllButLastSpecifier);
573     assertThrown("A positioned specifier %3$s mixed with unpositioned %d specifier".disableAllButLastSpecifier);
574 }
575 
576 /**
577 Code to be mixed in at the top of your `main()` function.
578 
579 Examples:
580 ---
581 void main()
582 {
583     import gettext;
584     mixin(gettext.main);
585 
586     // Your code.
587 }
588 ---
589 */
590 enum main = q{
591     version (xgettext)
592     {
593         if (scan) // Prevent unreachable code warning after mixin.
594         {
595             import std.traits : ReturnType;
596             static if (is (ReturnType!main == void))
597                 return;
598             else
599                 return 0;
600         }
601     }
602 };
603 
604 import mofile;
605 
606 private MoFile currentLanguage;
607 
608 /**
609 Collect a list of available `*.mo` files.
610 
611 If no `moPath` is given, files are searched inside the `mo` folder assumed
612 to exist besides the file location of the running executable.
613 */
614 string[] availableLanguages(string moPath = null)
615 {
616     import std.algorithm: map;
617     import std.array : array;
618     import std.file : exists, isDir, dirEntries, SpanMode;
619     import std.path : buildPath, dirName;
620 
621     if (moPath == null)
622     {
623         import core.runtime : Runtime;
624         moPath = buildPath(Runtime.args[0].dirName, "mo");
625     }
626 
627     if (moPath.exists && moPath.isDir)
628         return dirEntries(moPath, "*.mo", SpanMode.shallow).map!(a => a.name).array;
629 
630     return null;
631 }
632 
633 /**
634 Returns the language code for the current language.
635 */
636 string languageCode() @safe
637 {
638     import std.string : lineSplitter;
639     import std.algorithm : find, startsWith;
640     auto l = currentLanguage.header.lineSplitter.find!(a => a.startsWith("Language: "));
641     return l.empty ? "Default" : l.front["Language: ".length .. $];
642 }
643 
644 /**
645 Returns the language code for the translation contained in `moFile`.
646 */
647 string languageCode(string moFile) @safe
648 {
649     import std.string : lineSplitter;
650     import std.algorithm : find, startsWith;
651     auto l = MoFile(moFile).header.lineSplitter.find!(a => a.startsWith("Language: "));
652     return l.empty ? "Undefined" : l.front["Language: ".length .. $];
653 }
654 
655 /**
656 Switch to the language contained in `moFile`.
657 */
658 void selectLanguage(string moFile) @safe
659 {
660     import std.file : exists, isFile;
661 
662     currentLanguage = moFile.exists && moFile.isFile ? MoFile(moFile) : MoFile();
663 }