1 /**
2 Internationalization compatible with GNU gettext.
3 Authors:
4 $(LINK2 https://github.com/veelo, Bastiaan Veelo)
5 Copyright:
6 SARC B.V., 2022
7 License:
8 $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
9 See_Also:
10 $(LINK2 https://www.gnu.org/software/gettext/, GNU gettext utilities)
11 
12 
13 Translatable strings are marked by instantiating the `tr` template, like so:
14 ```
15 writeln(tr!"Translatable message");
16 ```
17 
18 If you'd rather use an underscore to mark translatable strings, [as the GNU
19 gettext documentation suggests](https://www.gnu.org/software/gettext/manual/html_node/Mark-Keywords.html),
20 you can use an alias:
21 ```
22 import gettext : _ = tr;    // Customary in GNU software.
23 writeln(_!"Translatable message");
24 ```
25 */
26 
27 module gettext;
28 
29 /// Optional attribute categories.
30 enum Tr { note, context }
31 
32 version (xgettext) // String extraction mode.
33 {
34     bool scan()
35     {
36         import std.getopt;
37         import std.path : baseName, buildPath, setExtension;
38         import core.runtime : Runtime;
39 
40         auto args = Runtime.args;
41         potFile = buildPath("po", args[0].baseName);
42 
43         auto helpInformation = getopt(args,
44                                       "output|o", "The path for the PO template file.", &potFile);
45         if (helpInformation.helpWanted)
46         {
47             ()@trusted{
48                 defaultGetoptPrinter("Usage:\n\tdub run --config=xgettext [-- <options>]\nOptions:", helpInformation.options);
49             }();
50         }
51         else
52             writePOT(potFile.setExtension("pot"));
53         return args.length > 0; // Always true, but the compiler has no idea.
54     }
55 
56     import std.typecons : Tuple;
57     import std.array : join;
58     import std.ascii : newline;
59 
60     enum Format {plain, c}
61     alias Key = Tuple!(string, "singular",
62                        string, "plural",
63                        Format, "format",
64                        string, "context");
65     private string[][Key] translatableStrings;
66     private string[][Key] comments;
67 
68     string potFile;
69 
70     private void writePOT(string potFile) @safe
71     {
72         import std.algorithm : cmp, map, sort;
73         import std.file : mkdirRecurse, write;
74         import std.path : baseName, dirName;
75         import std.stdio;
76 
77         string header() @safe
78         {
79             import std.exception : ifThrown;
80 
81             import std.array : join;
82 
83             import std.json, std.process;
84 
85             string rootPackage = potFile.baseName;
86 
87             JSONValue json;
88             auto piped = pipeProcess(["dub", "describe"], Redirect.stdout);
89             scope (exit) piped.pid.wait;
90             json = ()@trusted{ return piped.stdout.byLine.join.parseJSON; }();
91             rootPackage = json["rootPackage"].str.ifThrown!JSONException(potFile.baseName);
92             foreach (_package; json["packages"].arrayNoRef)
93                 if (_package["name"].str == rootPackage)
94                 {
95                     json = _package;
96                     break;
97                 }
98 
99             string thisYear()
100             {
101                 return __DATE__[$-4 .. $];
102             }
103             string title()
104             {
105                 return "# PO Template for " ~ rootPackage ~ ".";
106             }
107             string copyright()
108             {
109                 return ("# " ~ json["copyright"].str)
110                     .ifThrown!JSONException("# Copyright © YEAR THE PACKAGE'S COPYRIGHT HOLDER");
111             }
112             string license()
113             {
114                 return ("# This file is distributed under the " ~ json["license"].str ~ " license.")
115                     .ifThrown!JSONException("# This file is distributed under the same license as the " ~ rootPackage ~ " package.");
116             }
117             string author()
118             {
119                 return (){ return json["authors"].arrayNoRef.map!(a => "# " ~ a.str ~ ", " ~ thisYear ~ ".").join(newline); }()
120                     .ifThrown!JSONException("# FIRST AUTHOR <EMAIL@ADDRESS>, " ~ thisYear ~ ".");
121             }
122             string idVersion()
123             {
124                 auto gitResult = execute(["git", "describe"]);
125                 auto _version = gitResult.status == 0 ? gitResult.output : "PACKAGE VERSION";
126                 return (`"Project-Id-Version: ` ~ _version ~ `\n"`);
127             }
128             string bugs()
129             {
130                 return `"Report-Msgid-Bugs-To: \n"`;
131             }
132             string creationDate()
133             {
134                 import std.datetime;
135                 return `"POT-Creation-Date: ` ~ Clock.currTime.toUTC.toISOExtString() ~ `\n"`;
136             }
137             return [title, copyright, license, author, `#`, `#, fuzzy`, `msgid ""`, `msgstr ""`,
138                     idVersion, bugs, creationDate,
139                     `"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"`,
140                     `"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"`,
141                     `"Language-Team: LANGUAGE <LL@li.org>\n"`,
142                     `"Language: \n"`,
143                     `"MIME-Version: 1.0\n"`,
144                     `"Content-Type: text/plain; charset=UTF-8\n"`,
145                     `"Content-Transfer-Encoding: 8bit\n"`,
146                     ``, ``].join(newline);
147         }
148         mkdirRecurse(potFile.dirName);
149         ()@trusted {
150             translatableStrings.rehash;
151             comments.rehash;
152         }();
153         write(potFile, header ~ translatableStrings.keys
154               .sort!((a, b) => cmp(translatableStrings[a][0], translatableStrings[b][0]) < 0)
155               .map!(key => messageFromKey(key)).join(newline));
156         writeln(potFile ~ " generated.");
157     }
158 
159     string stringify(string str) @safe pure
160     {
161         import std.conv : text;
162 
163         return text([str])[1 .. $-1];
164     }
165 
166     string messageFromKey(Key key) @safe
167     {
168         string message;
169         if (auto c = key in comments)
170             foreach (comment; *c)
171                 message ~= `#. ` ~ comment ~ newline;
172         message ~= `#: ` ~ translatableStrings[key].join(" ") ~ newline;
173         if (key.format == Format.c)
174             message ~= `#, c-format` ~ newline;
175         if (key.context != null)
176             message ~= `msgctxt ` ~ key.context.stringify ~ newline;
177         if (key.plural == null)
178         {
179             message ~= `msgid ` ~ key.singular.stringify ~ newline ~
180                        `msgstr ""` ~ newline;
181         }
182         else
183         {
184             message ~= `msgid ` ~ key.singular.stringify ~ newline ~
185                        `msgid_plural ` ~ key.plural.stringify ~ newline ~
186                        `msgstr[0] ""` ~ newline ~
187                        `msgstr[1] ""` ~ newline;
188         }
189         return message;
190     }
191 
192     template tr(string singular, string[Tr] attributes = null,
193                 int line = __LINE__, string file = __FILE__, string mod = __MODULE__, string func = __FUNCTION__, Args...)
194     {
195         alias tr = tr!(singular, null, attributes,
196                        line, file, mod, func, Args);
197     }
198 
199     template tr(string singular, string plural, string[Tr] attributes = null,
200                 int line = __LINE__, string file = __FILE__, string mod = __MODULE__, string func = __FUNCTION__, Args...)
201     {
202         static struct StrInjector
203         {
204             static this()
205             {
206                 string reference()
207                 {
208                     import std.conv : to;
209                     import std.array : join;
210                     import std.algorithm : commonPrefix;
211                     import std.path : buildPath, pathSplitter;
212 
213                     static if (func.length == 0)
214                         return file.pathSplitter.join("/") ~ ":" ~ line.to!string;
215                     else
216                         return file.pathSplitter.join("/") ~ ":" ~ line.to!string ~ "(" ~ func[commonPrefix(func, mod).length + 1 .. $] ~ ")";
217                 }
218                 Format format()
219                 {
220                     static if (Args.length > 0 || singular.hasFormatSpecifiers || (plural && plural.hasFormatSpecifiers))
221                         return Format.c;
222                     else
223                         return Format.plain;
224                 }
225                 string context()
226                 {
227                     if (auto c = Tr.context in attributes)
228                         return *c;
229                     return null;
230                 }
231                 translatableStrings.require(Key(singular, plural, format, context)) ~= reference;
232                 if (auto c = Tr.note in attributes)
233                    comments.require(Key(singular, plural, format, context)) ~= *c;
234             }
235         }
236         static if (plural == null)
237             enum tr = TranslatableString(singular);
238         else
239             enum tr = TranslatableStringPlural(singular, plural);
240     }
241 
242     private bool hasFormatSpecifiers(string fmt) pure @safe
243     {
244         import std.format : FormatSpec;
245 
246         static void ns(const(char)[] arr) {} // the simplest output range
247         auto nullSink = &ns;
248         return FormatSpec!char(fmt).writeUpToNextSpec(nullSink);
249     }
250     unittest 
251     {
252         assert ("On %2$s I eat %3$s and walk for %1$d hours.".hasFormatSpecifiers);
253         assert ("On %%2$s I eat %%3$s and walk for %1$d hours.".hasFormatSpecifiers);
254         assert (!"On %%2$s I eat %%3$s and walk for hours.".hasFormatSpecifiers);
255     }
256 }
257 else // Translation mode.
258 {
259     template tr(string singular, string[Tr] attributes = null)
260     {
261         enum tr = TranslatableString(singular);
262     }
263     template tr(string singular, string plural, string[Tr] attributes = null)
264     {
265         enum tr = TranslatableStringPlural(singular, plural);
266     }
267 }
268 import std.format : format, FormatException, FormatSpec;
269 version (docs) {
270     /**
271     Translate `message`.
272 
273     This does *not* instantiate a new function for every marked string
274     (the signature is fabricated for the sake of documentation).
275 
276     Returns: The translation of `message` if one exists in the selected
277     language, or `message` otherwise.
278     See_Also: [selectLanguage]
279 
280     Examples:
281     ```
282     writeln(tr!"Translatable message");
283     ```
284     */
285     string tr(string message)() {};
286     /**
287     Translate a message in the correct plural form.
288 
289     This does *not* instantiate a new function for every marked string
290     (the signature is fabricated for the sake of documentation).
291 
292     The first argument should be in singular form, the second in plural
293     form. Note that the format specifier `%d` is optional.
294 
295     Returns: The translation if one exists in the selected
296     language, or the corresponding original otherwise.
297     See_Also: [selectLanguage]
298 
299     Examples:
300     ```
301     writeln(tr!("There is a goose!", "There are %d geese!")(n));
302     ```
303     */
304     string tr(string singular, string plural)(size_t n) {};
305 }
306 /*
307 This struct+template trick allows the string to be passed as template parameter without instantiating
308 a separate function for every string. https://forum.dlang.org/post/t8pqvg$20r0$1@digitalmars.com
309 */
310 @safe struct TranslatableString
311 {
312     private immutable(string)[] seq;
313     this (string str) nothrow
314     {
315         seq = [str];
316     }
317     this (string[] seq) nothrow
318     {
319         this.seq = seq.idup;
320     }
321     this (immutable(string)[] seq) nothrow
322     {
323         this.seq = seq;
324     }
325     string gettext() const
326     {
327         import std.algorithm : map;
328         import std.array : join;
329 
330         return seq.map!(a => currentLanguage.gettext(a)).join;
331     }
332     alias gettext this;
333     string toString() const // Called when a tr!"" literal or constant occurs in a writeln().
334     {
335         return gettext;
336     }
337     TranslatableString opBinary(string op : "~", RHS)(RHS rhs) nothrow
338     {
339         import std.traits : Unconst;
340 
341         static if (is (RHS == TranslatableString))
342             return TranslatableString(seq ~ rhs.seq);
343         else static if (is (Unconst!RHS == TranslatableString))
344             return TranslatableString(seq ~ rhs.seq.dup);
345         // TODO Add a reserved context so that ordinary strings don't accidentally get translated.
346         else static if (is (RHS == string))
347             return TranslatableString(seq ~ rhs);
348         else static if (is (Unconst!RHS == char))
349             return TranslatableString(seq ~ [rhs].idup);
350         else
351             static assert (false, "Need implementation for " ~ RHS.stringof);
352     }
353     TranslatableString opBinaryRight(string op : "~", LHS)(LHS lhs) nothrow
354     {
355         import std.traits : Unconst;
356 
357         static if (is (LHS == TranslatableString))
358             return TranslatableString(lhs.seq ~ seq);
359         static if (is (Unconst!LHS == TranslatableString))
360             return TranslatableString(lhs.seq.dup ~ seq);
361         // TODO Add a reserved context so that ordinary strings don't accidentally get translated.
362         else static if (is (LHS == string))
363             return TranslatableString([lhs] ~ seq);
364         else static if (is (LHS == char[]))
365             return TranslatableString([lhs.idup] ~ seq);
366         else static if (is (LHS == char))
367             return TranslatableString([[lhs.idup]] ~ seq);
368         else
369             static assert (false, "Need implementation for " ~ LHS.stringof);
370     }
371 }
372 @safe struct TranslatableStringPlural
373 {
374     string str, strpl;
375     this(string str, string strpl)
376     {
377         this.str = str;
378         this.strpl = strpl;
379     }
380     string opCall(size_t number) const
381     {
382         import std.algorithm : max;
383 
384         const n = cast (int) (number > int.max ? (number % 1000000) + 1000000 : number);
385         const trans =  currentLanguage.ngettext(str, strpl, n);
386         if (countFormatSpecifiers(trans) == countFormatSpecifiers(strpl))
387         {
388             try
389                 return format(trans.disableAllButLastSpecifier, n);
390             catch(FormatException e)
391             {
392                 debug throw(e);
393                 return strpl;   // Fall back on untranslated message.
394             }
395         }
396         else
397             return trans;
398     }
399 }
400 
401 private int countFormatSpecifiers(string fmt) pure @safe
402 {
403     static void ns(const(char)[] arr) {} // the simplest output range
404     auto nullSink = &ns;
405     int count = 0;
406     auto f = FormatSpec!char(fmt);
407     while (f.writeUpToNextSpec(nullSink))
408         count++;
409     return count;
410 }
411 unittest 
412 {
413     assert ("On %2$s I eat %3$s and walk for %1$d hours.".countFormatSpecifiers == 3);
414     assert ("On %%2$s I eat %%3$s and walk for %1$d hours.".countFormatSpecifiers == 1);
415 }
416 
417 private immutable(Char)[] disableAllButLastSpecifier(Char)(const Char[] inp) @safe
418 {
419     import std.array : Appender;
420     import std.conv : to;
421     import std.exception : enforce;
422     import std.typecons : tuple;
423 
424     enum Mode {undefined, inSequence, outOfSequence}
425     Mode mode = Mode.undefined;
426 
427     Appender!(Char[]) outp;
428     outp.reserve(inp.length + 10);
429     // Traverse specs, disable all of them, note where the highest index is. Re-enable that one.
430     size_t lastSpecIndex = 0, highestSpecIndex = 0, highestSpecPos = 0, specs = 0;
431     auto f = FormatSpec!Char(inp);
432     while (f.trailing.length > 0)
433     {
434         if (f.writeUpToNextSpec(outp))
435         {
436             // Mode check.
437             if (mode == Mode.undefined)
438                 mode = f.indexStart > 0 ? Mode.outOfSequence : Mode.inSequence;
439             else
440                 enforce!FormatException( mode == Mode.inSequence && f.indexStart == 0 ||
441                                         (mode == Mode.outOfSequence && f.indexStart != lastSpecIndex),
442                         `Cannot mix specifiers with and without a position argument in "` ~ inp ~ `"`);
443             // Track the highest.
444             if (f.indexStart == 0)
445                 highestSpecPos = outp[].length + 1;
446             else
447                 if (f.indexStart > highestSpecIndex)
448                 {
449                     highestSpecIndex = f.indexStart;
450                     highestSpecPos = outp[].length + 1;
451                 }
452             // disable
453             auto curFmtSpec = inp[outp[].length - specs .. $ - f.trailing.length];
454             outp ~= '%'.to!Char ~ curFmtSpec;
455             lastSpecIndex = f.indexStart;
456             specs++;
457         }
458 
459     }
460     return mode == Mode.inSequence ?
461         (outp[][0 .. highestSpecPos] ~ outp[][highestSpecPos + 1 .. $]).idup :
462         (outp[][0 .. highestSpecPos] ~ outp[][highestSpecPos + highestSpecIndex.to!string.length + 2 .. $]).idup;
463 }
464 unittest
465 {
466     import std.exception;
467 
468     assert ("Я считаю %d яблоко.".disableAllButLastSpecifier ==
469             "Я считаю %d яблоко.");
470     assert ("Last %s, in %s, I ate %d muffins".disableAllButLastSpecifier ==
471             "Last %%s, in %%s, I ate %d muffins");
472     assert ("I ate %3$d muffins on %1$s in %2$s.".disableAllButLastSpecifier ==
473             "I ate %d muffins on %%1$s in %%2$s.");
474     assertThrown("An unpositioned %d specifier mixed with positioned specifier %3$s".disableAllButLastSpecifier);
475     assertThrown("A positioned specifier %3$s mixed with unpositioned %d specifier".disableAllButLastSpecifier);
476 }
477 
478 /**
479 Code to be mixed in at the top of your `main()` function.
480 
481 Examples:
482 ```
483 void main()
484 {
485 import gettext;
486 mixin(gettext.main);
487 
488 // Your code.
489 }
490 ```
491 */
492 enum main = q{
493     version (xgettext)
494     {
495         if (scan) // Prevent unreachable code warning after mixin.
496         {
497             import std.traits : ReturnType;
498             static if (is (ReturnType!main == void))
499                 return;
500             else
501                 return 0;
502         }
503     }
504 };
505 
506 import mofile;
507 
508 private MoFile currentLanguage;
509 
510 /**
511 Collect a list of available `*.mo` files.
512 
513 If no `moPath` is given, files are searched inside the `mo` folder assumed
514 to exist besides the file location of the running executable.
515 */
516 string[] availableLanguages(string moPath = null)
517 {
518     import std.algorithm: map;
519     import std.array : array;
520     import std.file : exists, isDir, dirEntries, SpanMode;
521     import std.path : buildPath, dirName;
522 
523     if (moPath == null)
524     {
525         import core.runtime : Runtime;
526         moPath = buildPath(Runtime.args[0].dirName, "mo");
527     }
528 
529     if (moPath.exists && moPath.isDir)
530         return dirEntries(moPath, "*.mo", SpanMode.shallow).map!(a => a.name).array;
531 
532     return null;
533 }
534 
535 /**
536 Returns the language code for the current language.
537 */
538 string languageCode() @safe
539 {
540     import std.string : lineSplitter;
541     import std.algorithm : find, startsWith;
542     auto l = currentLanguage.header.lineSplitter.find!(a => a.startsWith("Language: "));
543     return l.empty ? "Default" : l.front["Language: ".length .. $];
544 }
545 
546 /**
547 Returns the language code for the translation contained in `moFile`.
548 */
549 string languageCode(string moFile) @safe
550 {
551     import std.string : lineSplitter;
552     import std.algorithm : find, startsWith;
553     auto l = MoFile(moFile).header.lineSplitter.find!(a => a.startsWith("Language: "));
554     return l.empty ? "Undefined" : l.front["Language: ".length .. $];
555 }
556 
557 /**
558 Switch to the language contained in `moFile`.
559 */
560 void selectLanguage(string moFile) @safe
561 {
562     import std.file : exists, isFile;
563 
564     currentLanguage = moFile.exists && moFile.isFile ? MoFile(moFile) : MoFile();
565 }