Update for Unicode 16 / TR46 rev 33

In addition to the dependency update on tr46, this includes a series of recent URL Standard changes:

* whatwg/url@cd8f1d6
* whatwg/url@a6e4492
* whatwg/url@c3d173f
* whatwg/url@7f3e3b6

Closes #291.

Author: domenic

README.md

@@ -4,7 +4,7 @@ whatwg-url is a full implementation of the WHATWG [URL Standard](https://url.spe
 
 ## Specification conformance
 
-whatwg-url is currently up to date with the URL spec up to commit [da212c9](https://github.com/whatwg/url/commit/da212c98f0bba9c35aec3728bbcc13f8f3a7eb52).
+whatwg-url is currently up to date with the URL spec up to commit [cfae7c4](https://github.com/whatwg/url/commit/cfae7c4ed1851886390e132fc336b653fada9123).
 
 For `file:` URLs, whose [origin is left unspecified](https://url.spec.whatwg.org/#concept-url-origin), whatwg-url chooses to use a new opaque origin (which serializes to `"null"`).
 

lib/url-state-machine.js

@@ -344,10 +344,6 @@ function parseHost(input, isOpaque = false) {
     return failure;
   }
 
-  if (containsForbiddenDomainCodePoint(asciiDomain)) {
-    return failure;
-  }
-
   if (endsInANumber(asciiDomain)) {
     return parseIPv4(asciiDomain);
   }
@@ -429,15 +425,26 @@ function serializeHost(host) {
 
 function domainToASCII(domain, beStrict = false) {
   const result = tr46.toASCII(domain, {
+    checkHyphens: beStrict,
     checkBidi: true,
-    checkHyphens: false,
     checkJoiners: true,
     useSTD3ASCIIRules: beStrict,
-    verifyDNSLength: beStrict
+    transitionalProcessing: false,
+    verifyDNSLength: beStrict,
+    ignoreInvalidPunycode: false
   });
-  if (result === null || result === "") {
+  if (result === null) {
     return failure;
   }
+
+  if (!beStrict) {
+    if (result === "") {
+      return failure;
+    }
+    if (containsForbiddenDomainCodePoint(result)) {
+      return failure;
+    }
+  }
   return result;
 }
 

package-lock.json

@@ -12,7 +12,7 @@
   "license": "MIT",
   "repository": "jsdom/whatwg-url",
   "dependencies": {
-    "tr46": "^5.0.0",
+    "tr46": "^5.1.0",
     "webidl-conversions": "^7.0.0"
   },
   "devDependencies": {

package.json

@@ -13,7 +13,7 @@ const path = require("path");
 // 1. Go to https://github.com/web-platform-tests/wpt/tree/master/url
 // 2. Press "y" on your keyboard to get a permalink
 // 3. Copy the commit hash
-const commitHash = "a9fe2e77b64b0ce9aefb2ed4331c52bfa44fced4";
+const commitHash = "7c9c41aedbcd49bf96fb9e35a133293b3977b83d";
 
 const urlPrefix = `https://raw.githubusercontent.com/web-platform-tests/wpt/${commitHash}/url/`;
 const targetDir = path.resolve(__dirname, "..", "test", "web-platform-tests");
@@ -25,7 +25,8 @@ const resources = [
   "resources/toascii.json",
   "resources/urltestdata.json",
   "resources/urltestdata-javascript-only.json",
-  "resources/IdnaTestV2.json"
+  "resources/IdnaTestV2.json",
+  "resources/IdnaTestV2-removed.json"
 ];
 
 // These tests we can download and run directly in /test/web-platform.js.
@@ -54,6 +55,7 @@ exports.directlyRunnableTests = [
 // can distinguish.
 exports.resourceDependentTests = [
   "IdnaTestV2.window.js",
+  "IdnaTestV2-removed.window.js",
   "url-constructor.any.js",
   "url-origin.any.js",
   "url-setters.any.js"

scripts/get-latest-platform-tests.js

@@ -11,6 +11,7 @@ const { utf8PercentEncodeString, isSpecialQueryPercentEncode } = require("../lib
 const { URL, URLSearchParams } = require("..");
 
 const idnaTestV2Data = require("./web-platform-tests/resources/IdnaTestV2.json");
+const idnaTestV2RemovedData = require("./web-platform-tests/resources/IdnaTestV2-removed.json");
 const urlTestData = require("./web-platform-tests/resources/urltestdata.json");
 const urlTestDataJavaScriptOnly = require("./web-platform-tests/resources/urltestdata-javascript-only.json");
 const settersData = require("./web-platform-tests/resources/setters_tests.json");
@@ -29,6 +30,7 @@ describe("Data file-based web platform tests", () => {
       resourceDependentTests,
       [
         "IdnaTestV2.window.js",
+        "IdnaTestV2-removed.window.js",
         "url-constructor.any.js",
         "url-origin.any.js",
         "url-setters.any.js"
@@ -41,6 +43,10 @@ describe("Data file-based web platform tests", () => {
     sandbox.runTests(idnaTestV2Data);
   });
 
+  runWPT("IdnaTestV2-removed.window.js", sandbox => {
+    sandbox.runTests(idnaTestV2RemovedData);
+  });
+
   runWPT("url-constructor.any.js", sandbox => {
     sandbox.runURLTests(urlTestData);
     sandbox.runURLTests(urlTestDataJavaScriptOnly);