Internationalized domain name (IDN) support (#2286)

* Internationalized domain name (IDN) support

* Documentation fix

* Tests for base_uri

* PHPStan and CI fixes

* cs

composer.json

@@ -32,7 +32,8 @@
         "psr/log": "^1.1"
     },
     "suggest": {
-        "psr/log": "Required for using the Log middleware"
+        "psr/log": "Required for using the Log middleware",
+        "ext-intl": "Required for Internationalized Domain Name (IDN) support"
     },
     "config": {
         "sort-packages": true

docs/request-options.rst

@@ -553,6 +553,31 @@ http_errors
     default when creating a handler with ``GuzzleHttp\default_handler``.
 
 
+idn_conversion
+---
+
+:Summary: Internationalized Domain Name (IDN) support (enabled by default if
+    ``intl`` extension is available).
+:Types:
+    - bool
+    - int
+:Default: ``true`` if ``intl`` extension is available, ``false`` otherwise
+:Constant: ``GuzzleHttp\RequestOptions::IDN_CONVERSION``
+
+.. code-block:: php
+
+    $client->request('GET', 'https://яндекс.рф');
+    // яндекс.рф is translated to xn--d1acpjx3f.xn--p1ai before passing it to the handler
+
+    $res = $client->request('GET', 'https://яндекс.рф', ['idn_conversion' => false]);
+    // The domain part (яндекс.рф) stays unmodified
+
+Enables/disables IDN support, can also be used for precise control by combining
+IDNA_* constants (except IDNA_ERROR_*), see ``$options`` parameter in
+`idn_to_ascii() <https://www.php.net/manual/en/function.idn-to-ascii.php>`_
+documentation for more details.
+
+
 json
 ----
 

phpstan-baseline.neon

@@ -110,11 +110,6 @@ parameters:
 			count: 1
 			path: src/Client.php
 
-		-
-			message: "#^PHPDoc tag @throws with type GuzzleHttp\\\\InvalidArgumentException is not subtype of Throwable$#"
-			count: 1
-			path: src/Client.php
-
 		-
 			message: "#^Method GuzzleHttp\\\\ClientInterface\\:\\:send\\(\\) has parameter \\$options with no value type specified in iterable type array\\.$#"
 			count: 1

phpstan.neon.dist

@@ -5,3 +5,4 @@ parameters:
     level: max
     paths:
       - src
+    bootstrap: tests/bootstrap-phpstan.php

src/Client.php

@@ -2,6 +2,7 @@
 namespace GuzzleHttp;
 
 use GuzzleHttp\Cookie\CookieJar;
+use GuzzleHttp\Exception\InvalidArgumentException;
 use GuzzleHttp\Promise;
 use GuzzleHttp\Psr7;
 use Psr\Http\Message\RequestInterface;
@@ -214,6 +215,38 @@ class Client implements ClientInterface
             $uri = Psr7\UriResolver::resolve(Psr7\uri_for($config['base_uri']), $uri);
         }
 
+        if ($uri->getHost() && isset($config['idn_conversion']) && ($config['idn_conversion'] !== false)) {
+            $idnOptions = ($config['idn_conversion'] === true) ? IDNA_DEFAULT : $config['idn_conversion'];
+
+            $asciiHost = idn_to_ascii($uri->getHost(), $idnOptions, INTL_IDNA_VARIANT_UTS46, $info);
+            if ($asciiHost === false) {
+                $errorBitSet = isset($info['errors']) ? $info['errors'] : 0;
+
+                $errorConstants = array_filter(array_keys(get_defined_constants()), function ($name) {
+                    return substr($name, 0, 11) === 'IDNA_ERROR_';
+                });
+
+                $errors = [];
+                foreach ($errorConstants as $errorConstant) {
+                    if ($errorBitSet & constant($errorConstant)) {
+                        $errors[] = $errorConstant;
+                    }
+                }
+
+                $errorMessage = 'IDN conversion failed';
+                if ($errors) {
+                    $errorMessage .= ' (errors: ' . implode(', ', $errors) . ')';
+                }
+
+                throw new InvalidArgumentException($errorMessage);
+            } else {
+                if ($uri->getHost() !== $asciiHost) {
+                    // Replace URI only if the ASCII version is different
+                    $uri = $uri->withHost($asciiHost);
+                }
+            }
+        }
+
         return $uri->getScheme() === '' && $uri->getHost() !== '' ? $uri->withScheme('http') : $uri;
     }
 
@@ -233,12 +266,15 @@ class Client implements ClientInterface
             'cookies'         => false
         ];
 
+        // idn_to_ascii() is a part of ext-intl and might be not available
+        $defaults['idn_conversion'] = function_exists('idn_to_ascii');
+
         // Use the standard Linux HTTP_PROXY and HTTPS_PROXY if set.
 
         // We can only trust the HTTP_PROXY environment variable in a CLI
         // process due to the fact that PHP has no reliable mechanism to
         // get environment variables that start with "HTTP_".
-        if (php_sapi_name() == 'cli' && getenv('HTTP_PROXY')) {
+        if (php_sapi_name() === 'cli' && getenv('HTTP_PROXY')) {
             $defaults['proxy']['http'] = getenv('HTTP_PROXY');
         }
 

src/RequestOptions.php

@@ -132,6 +132,14 @@ final class RequestOptions
      */
     const HTTP_ERRORS = 'http_errors';
 
+    /**
+     * idn: (bool|int, default=true) A combination of IDNA_* constants for
+     * idn_to_ascii() PHP's function (see "options" parameter). Set to false to
+     * disable IDN support completely, or to true to use the default
+     * configuration (IDNA_DEFAULT constant).
+     */
+    const IDN_CONVERSION = 'idn_conversion';
+
     /**
      * json: (mixed) Adds JSON data to a request. The provided value is JSON
      * encoded and a Content-Type header of application/json will be added to

tests/ClientTest.php

@@ -711,4 +711,84 @@ class ClientTest extends TestCase
 
         self::assertSame($responseBody, $response->getBody()->getContents());
     }
+
+    public function testIdnSupportDefaultValue()
+    {
+        $mockHandler = new MockHandler([new Response()]);
+        $client = new Client(['handler' => $mockHandler]);
+
+        $config = $client->getConfig();
+
+        if (extension_loaded('intl')) {
+            self::assertTrue($config['idn_conversion']);
+        } else {
+            self::assertFalse($config['idn_conversion']);
+        }
+    }
+
+    public function testIdnIsTranslatedToAsciiWhenConversionIsEnabled()
+    {
+        if (!extension_loaded('intl')) {
+            self::markTestSkipped('intl PHP extension is not loaded');
+        }
+        $mockHandler = new MockHandler([new Response()]);
+        $client = new Client(['handler' => $mockHandler]);
+
+        $client->request('GET', 'https://яндекс.рф/images', ['idn_conversion' => true]);
+
+        $request = $mockHandler->getLastRequest();
+
+        self::assertSame('https://xn--d1acpjx3f.xn--p1ai/images', (string) $request->getUri());
+        self::assertSame('xn--d1acpjx3f.xn--p1ai', (string) $request->getHeaderLine('Host'));
+    }
+
+    public function testIdnStaysTheSameWhenConversionIsDisabled()
+    {
+        $mockHandler = new MockHandler([new Response()]);
+        $client = new Client(['handler' => $mockHandler]);
+
+        $client->request('GET', 'https://яндекс.рф/images', ['idn_conversion' => false]);
+
+        $request = $mockHandler->getLastRequest();
+
+        self::assertSame('https://яндекс.рф/images', (string) $request->getUri());
+        self::assertSame('яндекс.рф', (string) $request->getHeaderLine('Host'));
+    }
+
+    /**
+     * @expectedException \GuzzleHttp\Exception\InvalidArgumentException
+     * @expectedExceptionMessage IDN conversion failed (errors: IDNA_ERROR_LEADING_HYPHEN)
+     */
+    public function testExceptionOnInvalidIdn()
+    {
+        if (!extension_loaded('intl')) {
+            self::markTestSkipped('intl PHP extension is not loaded');
+        }
+        $mockHandler = new MockHandler([new Response()]);
+        $client = new Client(['handler' => $mockHandler]);
+
+        $client->request('GET', 'https://-яндекс.рф/images', ['idn_conversion' => true]);
+    }
+
+    /**
+     * @depends testCanUseRelativeUriWithSend
+     * @depends testIdnSupportDefaultValue
+     */
+    public function testIdnBaseUri()
+    {
+        if (!extension_loaded('intl')) {
+            self::markTestSkipped('intl PHP extension is not loaded');
+        }
+
+        $mock = new MockHandler([new Response()]);
+        $client = new Client([
+            'handler'  => $mock,
+            'base_uri' => 'http://яндекс.рф',
+        ]);
+        self::assertSame('http://яндекс.рф', (string) $client->getConfig('base_uri'));
+        $request = new Request('GET', '/baz');
+        $client->send($request);
+        self::assertSame('http://xn--d1acpjx3f.xn--p1ai/baz', (string) $mock->getLastRequest()->getUri());
+        self::assertSame('xn--d1acpjx3f.xn--p1ai', (string) $mock->getLastRequest()->getHeaderLine('Host'));
+    }
 }

tests/bootstrap-phpstan.php

@@ -0,0 +1,9 @@
+<?php
+
+if (!defined('IDNA_DEFAULT')) {
+    define('IDNA_DEFAULT', 0);
+}
+
+if (!defined('INTL_IDNA_VARIANT_UTS46')) {
+    define('INTL_IDNA_VARIANT_UTS46', 1);
+}

tests/bootstrap.php

@@ -1,4 +1,9 @@
 <?php
+
+namespace {
+    setlocale(LC_ALL, 'C');
+}
+
 namespace GuzzleHttp\Test {
     require __DIR__ . '/../vendor/autoload.php';
     require __DIR__ . '/Server.php';