tls: match IPv6 hosts against IP-Address SANs

checkServerIdentity() stopped matching an IPv6 host against a matching
IP-Address SAN. The hostname is now run through domainToASCII() before
the net.isIP() gate, and domainToASCII('::1') === '' (an IPv6 literal is
not a domain), so net.isIP('') is 0, the IP-SAN branch is skipped, and
verification fails with "Cert does not contain a DNS name". IPv4 is
unaffected because dotted-decimal survives domainToASCII().

Match IP hosts against the original hostname instead of the IDNA-
normalized one. net.isIP() rejects non-ASCII input, so there is no IDNA
confusion to guard against for an IP literal; the normalized form is
still used for the DNS-name path.

Fixes: #64144
Signed-off-by: Pascal Garber <pascal@artandcode.studio>
PR-URL: #64145
Reviewed-By: Tim Perry <pimterry@gmail.com>

Author: JumpLink

lib/tls.js

@@ -427,8 +427,14 @@ exports.checkServerIdentity = function checkServerIdentity(hostname, cert) {
   let valid = false;
   let reason = 'Unknown reason';
 
-  if (net.isIP(hostnameASCIIWithoutFQDN)) {
-    valid = ips.includes(canonicalizeIP(hostnameASCIIWithoutFQDN));
+  // An IP literal must not be IDNA-normalized: domainToASCII() returns '' for
+  // an IPv6 literal (it is not a domain), so matching against the normalized
+  // host would skip IP-SAN matching for IPv6 entirely. Match IP hosts against
+  // the original hostname (net.isIP() rejects non-ASCII, so there is no IDNA
+  // confusion to guard against here); the normalized form is kept for the
+  // DNS-name path below.
+  if (net.isIP(hostname)) {
+    valid = ips.includes(canonicalizeIP(hostname));
     if (!valid) {
       reason =
         `IP: ${hostname} is not in the cert's list: ` + ips.join(', ');

test/parallel/test-tls-check-server-identity.js

@@ -100,6 +100,27 @@ const tests = [
     cert: { subject: { CN: '8.8.8.8' }, subjectaltname: 'IP Address:8.8.8.8' }
   },
 
+  // An "IP Address:" SAN also matches an IPv6 host. Regression test for the
+  // IDNA-normalization change: domainToASCII('::1') === '' (an IPv6 literal is
+  // not a domain), which made the normalized host skip IPv6 IP-SAN matching.
+  {
+    host: '::1',
+    cert: { subject: {}, subjectaltname: 'IP Address:::1' }
+  },
+
+  // IPv6 hosts and SANs are matched canonically.
+  {
+    host: '2001:db8::1',
+    cert: { subject: {}, subjectaltname: 'IP Address:2001:DB8:0:0:0:0:0:1' }
+  },
+
+  // A non-matching IPv6 "IP Address:" SAN is rejected.
+  {
+    host: '::1',
+    cert: { subject: {}, subjectaltname: 'IP Address:::2' },
+    error: 'IP: ::1 is not in the cert\'s list: ::2'
+  },
+
   // But not when it's a CIDR.
   {
     host: '8.8.8.8',