feat: Adding reference language feature

Author: kargnas

README.md

@@ -213,12 +213,8 @@ php artisan vendor:publish --provider="Kargnas\LaravelAiTranslator\ServiceProvid
 
 This will create a `config/ai-translator.php` file where you can modify the following settings:
 
-- `source_locale`: Change this to your default language in the Laravel project. The package will translate from this language.
-
 - `source_directory`: If you use a different directory for language files instead of the default `lang` directory, you can specify it here.
 
-- `chunk_size`: Set the number of strings to be translated in a single AI request. Higher values can significantly reduce API costs but may impact translation quality for very large chunks. Default is 10.
-
 - `ai`: Configure the AI provider, model, and API key here. Here are our recommendation for the best models:
 
   | Provider  | Model                          | Cost (I/O per 1M tokens) | Descrpition                                      |
@@ -247,9 +243,7 @@ Example configuration:
 <?php
 
 return [
-    'source_locale' => 'en',
     'source_directory' => 'lang',
-    'chunk_size' => 10,
 
     'ai' => [
         'provider' => 'openai', // or 'anthropic'

config/ai-translator.php

@@ -1,10 +1,7 @@
 <?php
 
 return [
-    'source_locale' => 'en',
     'source_directory' => 'lang',
-    // Translate strings in a batch. The higher, the cheaper.
-    'chunk_size' => 10,
 
     'ai' => [
 //        'provider' => 'anthropic',

src/AI/AIProvider.php

@@ -69,15 +69,22 @@ protected function getUserPrompt($replaces = []) {
         $replaces = array_merge($replaces, [
             'sourceLanguage' => $this->sourceLanguage,
             'targetLanguage' => $this->targetLanguage,
+            'filename' => $this->filename,
             'parentKey' => basename($this->filename, '.php'),
             'strings' => collect($this->strings)->map(function ($string, $key) {
                 if (is_string($string)) {
                     return "  - `{$key}`: \"\"\"{$string}\"\"\"";
                 } else {
                     $text = "  - `{$key}`: \"\"\"{$string['text']}\"\"\"";
-                    if ($string['context']) {
+                    if (isset($string['context'])) {
                         $text .= "\n    - Context: \"\"\"{$string['context']}\"\"\"";
                     }
+                    if (isset($string['references']) && sizeof($string['references']) > 0) {
+                        $text .= "\n    - References:";
+                        foreach ($string['references'] as $locale => $items) {
+                            $text .= "\n      - {$locale}: \"\"\"" . $items . "\"\"\"";
+                        }
+                    }
                     return $text;
                 }
             })->implode("\n"),
@@ -130,7 +137,7 @@ public function getTranslatedObjects(): array {
 
         // Fix Parent key issue
         $parentKey = basename($this->filename, '.php');
-        foreach($result as $item) {
+        foreach ($result as $item) {
             if (str_starts_with($item->key, $parentKey)) {
                 $item->key = str_replace($parentKey . '.', '', $item->key);
             }

src/AI/prompt-user.txt

@@ -1,11 +1,10 @@
 # Target String
-
 The most important mission: All strings should be translated into {targetLanguage} from {sourceLanguage}.
 - Source Language: `{sourceLanguage}`
 - Target Language: `{targetLanguage}`
 - Source filename: `{filename}`
 
 ## Context
 - Parent key (Only for your reference, don't put to the key): `{parentKey}`
-- The source strings: (e.g. key: """(value)"""), these are the strings that you should translate into the target language.
+- The source strings: (e.g. `key`: """(value)"""), these are the strings that you should translate into the target language.
 {strings}

src/Console/TranslateStrings.php

@@ -276,6 +276,7 @@ class TranslateStrings extends Command
     protected $sourceLocale;
     protected $sourceDirectory;
     protected $chunkSize;
+    protected array $referenceLocales = [];
 
     public function __construct() {
         parent::__construct();
@@ -286,10 +287,15 @@ public function __construct() {
     }
 
     public function handle() {
-        $this->sourceLocale = config('ai-translator.source_locale');
         $this->sourceDirectory = config('ai-translator.source_directory');
-        $this->chunkSize = config('ai-translator.chunk_size', 10);
 
+        $this->sourceLocale = $this->choiceLanguages("Choose a source language to translate from", false, 'en');
+
+        if ($this->ask('Do you want to add reference languages? (y/n)', 'n') === 'y') {
+            $this->referenceLocales = $this->choiceLanguages("Choose a language to reference when translating, preferably one that has already been vetted and translated to a high quality. You can select multiple languages via ',' (e.g. '1, 2')", true);
+        }
+
+        $this->chunkSize = $this->ask("Enter the chunk size for translation. Translate strings in a batch. The higher, the cheaper. (default: 10)", 10);
         $this->translate();
     }
 
@@ -391,6 +397,25 @@ protected static function getAdditionalRules($locale): array {
         return array_merge(static::getAdditionalRulesFromConfig($locale), static::getAdditionalRulesDefault($locale), static::getAdditionalRulesPlural($locale));
     }
 
+    public function choiceLanguages($question, $multiple, $default = null) {
+        $locales = $this->getExistingLocales();
+
+        $selectedLocales = $this->choice(
+            $question,
+            $locales,
+            $default,
+            3,
+            $multiple);
+
+        if (is_array($selectedLocales)) {
+            $this->info("Selected locales: " . implode(', ', $selectedLocales));
+        } else {
+            $this->info("Selected locale: " . $selectedLocales);
+        }
+
+        return $selectedLocales;
+    }
+
     public function translate() {
         $locales = $this->getExistingLocales();
         foreach ($locales as $locale) {
@@ -422,6 +447,17 @@ public function translate() {
                     })
                     ->toArray();
 
+                $referenceStringList = collect($this->referenceLocales)
+                    ->filter(fn($referenceLocale) => !in_array($referenceLocale, [$locale, $this->sourceLocale]))
+                    ->mapWithKeys(function ($referenceLocale) use ($file, $targetStringTransformer) {
+                        $referenceFile = $this->getOutputDirectoryLocale($referenceLocale) . '/' . basename($file);
+                        $referenceTransformer = new PHPLangTransformer($referenceFile);
+                        return [
+                            $referenceLocale => $referenceTransformer->flatten(),
+                        ];
+                    })
+                    ->toArray();
+
                 if (sizeof($sourceStringList) > 100) {
                     if (!$this->confirm("{$outputFile}, Strings: " . sizeof($sourceStringList) . " -> Many strings to translate. Could be expensive. Continue?")) {
                         $this->warn("Stopped translating!");
@@ -433,10 +469,21 @@ public function translate() {
                 // But also this will increase the speed of the translation, and quality of continuous translation
                 collect($sourceStringList)
                     ->chunk($this->chunkSize)
-                    ->each(function ($chunk) use ($locale, $file, $targetStringTransformer) {
+                    ->each(function ($chunk) use ($locale, $file, $targetStringTransformer, $referenceStringList) {
                         $translator = new AIProvider(
                             filename: $file,
-                            strings: $chunk->toArray(),
+                            strings: $chunk->mapWithKeys(function ($item, $key) use ($referenceStringList) {
+                                return [
+                                    $key => [
+                                        'text' => $item,
+                                        'references' => collect($referenceStringList)->map(function ($items) use ($key) {
+                                            return $items[$key] ?? "";
+                                        })->filter(function ($value) {
+                                            return strlen($value) > 0;
+                                        }),
+                                    ],
+                                ];
+                            })->toArray(),
                             sourceLanguage: static::getLanguageName($this->sourceLocale) ?? $this->sourceLocale,
                             targetLanguage: static::getLanguageName($locale) ?? $locale,
                             additionalRules: static::getAdditionalRules($locale),
@@ -455,14 +502,17 @@ public function translate() {
         }
     }
 
+    /**
+     * @return array|string[]
+     */
     public function getExistingLocales(): array {
         $root = $this->sourceDirectory;
         $directories = array_diff(scandir($root), ['.', '..']);
         // only directories
         $directories = array_filter($directories, function ($directory) use ($root) {
             return is_dir($root . '/' . $directory);
         });
-        return $directories;
+        return collect($directories)->values()->toArray();
     }
 
     public function getOutputDirectoryLocale($locale) {