Draft impl of idn-hostname validator. Add libraries to simplify work with unicode

Author: OptimumCode

README.md

@@ -389,6 +389,14 @@ You can see the results in the latest workflow execution.
 The update to Kotlin 1.9.22 came with an issue for JS incremental compilation.
 In case you see an error about main function that already bind please execute `clean` task.
 
+When you build project for **linux** target you might get an error about missing native library.
+This is because `de.cketti.unicode:kotlin-codepoints` requires this library to perform string normalization.
+This is needed to support `idn-hostname` format. Install this library with the following command:
+
+```bash
+sudo apt-get install -y libunistring-dev
+```
+
 ### Devcontainer
 
 Devcontainers is a cool feature. However, by default in Codespaces and DevPod you will use [VS Code](https://code.visualstudio.com/).

build.gradle.kts

@@ -77,6 +77,12 @@ kotlin {
       dependencies {
         api(libs.kotlin.serialization.json)
         implementation(libs.uri)
+        implementation("de.cketti.unicode:kotlin-codepoints:0.7.0") {
+          because("simplifies work with unicode codepoints")
+        }
+        implementation("com.doist.x:normalize:1.0.5") {
+          because("provides normalization required by IDN-hostname format")
+        }
       }
     }
     commonTest {

src/commonMain/kotlin/io/github/optimumcode/json/schema/internal/factories/general/FormatAssertionFactory.kt

@@ -17,6 +17,7 @@ import io.github.optimumcode.json.schema.internal.formats.DateFormatValidator
 import io.github.optimumcode.json.schema.internal.formats.DateTimeFormatValidator
 import io.github.optimumcode.json.schema.internal.formats.DurationFormatValidator
 import io.github.optimumcode.json.schema.internal.formats.HostnameFormatValidator
+import io.github.optimumcode.json.schema.internal.formats.IdnHostnameFormatValidator
 import io.github.optimumcode.json.schema.internal.formats.IpV4FormatValidator
 import io.github.optimumcode.json.schema.internal.formats.IpV6FormatValidator
 import io.github.optimumcode.json.schema.internal.formats.JsonPointerFormatValidator
@@ -70,6 +71,7 @@ internal sealed class FormatAssertionFactory(
         "ipv6" to IpV6FormatValidator,
         "uuid" to UuidFormatValidator,
         "hostname" to HostnameFormatValidator,
+        "idn-hostname" to IdnHostnameFormatValidator,
       )
   }
 }

src/commonMain/kotlin/io/github/optimumcode/json/schema/internal/formats/IdnHostnameFormatValidator.kt

@@ -0,0 +1,101 @@
+package io.github.optimumcode.json.schema.internal.formats
+
+import de.cketti.codepoints.codePointAt
+import de.cketti.codepoints.codePointBefore
+import io.github.optimumcode.json.schema.FormatValidationResult
+import io.github.optimumcode.json.schema.FormatValidator
+import io.github.optimumcode.json.schema.internal.hostname.Normalizer
+import io.github.optimumcode.json.schema.internal.hostname.Punycode
+import kotlin.text.CharCategory.COMBINING_SPACING_MARK
+import kotlin.text.CharCategory.ENCLOSING_MARK
+import kotlin.text.CharCategory.NON_SPACING_MARK
+
+internal object IdnHostnameFormatValidator : AbstractStringFormatValidator() {
+  override fun validate(value: String): FormatValidationResult {
+    if (value.isEmpty()) {
+      return FormatValidator.Invalid()
+    }
+    if (value.length == 1 && isLabelSeparator(value[0])) {
+      return FormatValidator.Valid()
+    }
+    var pointer = 0
+    while (pointer < value.length) {
+      val dot = findDot(value, pointer)
+      val label = value.substring(pointer, dot)
+      if (!isValidLabel(label)) {
+        return FormatValidator.Invalid()
+      }
+      pointer = dot + 1
+    }
+    return FormatValidator.Valid()
+  }
+
+  private fun isValidLabel(label: String): Boolean {
+    val unicode =
+      if (isACE(label)) {
+        Punycode.decode(label) ?: return false
+      } else {
+        label
+      }
+
+    if (!Normalizer.isNormalized(unicode)) {
+      return false
+    }
+
+    // https://datatracker.ietf.org/doc/html/rfc5891#section-4.2.3.1
+    if (unicode[0] == '-' || unicode.codePointBefore(unicode.length) == '-'.code) {
+      // cannot start or end with hyphen
+      return false
+    }
+
+    if (unicode.length >= 4 && hasTwoConsecutiveHyphens(unicode)) {
+      // cannot have to consecutive hyphens at 3 and 4 char position
+      return false
+    }
+
+    val firstCodePoint = unicode.codePointAt(0)
+    // https://datatracker.ietf.org/doc/html/rfc5891#section-4.2.3.2
+    if (isLeadingCombiningMark(firstCodePoint)) {
+      return false
+    }
+
+    // TODO: check common rules and specific depending on char direction
+    // TODO: encode using Punycode and check length
+
+    return true
+  }
+
+  private fun isLeadingCombiningMark(codePoint: Int): Boolean =
+    // I am not sure if this is correct
+    // code point might be converted to the char incorrectly
+    // need to have more tests on this
+    // TODO: add tests with code points greater than Char.MAX_VALUE
+    when (codePoint.toChar().category) {
+      NON_SPACING_MARK,
+      COMBINING_SPACING_MARK,
+      ENCLOSING_MARK,
+      -> true
+
+      else -> false
+    }
+
+  private fun hasTwoConsecutiveHyphens(value: String): Boolean =
+    value.codePointAt(2) == '-'.code && value.codePointAt(3) == '-'.code
+
+  private fun isACE(label: String): Boolean =
+    label.length > Punycode.PREFIX_SIZE && label.startsWith(Punycode.PREFIX_STRING)
+
+  private fun isLabelSeparator(c: Char): Boolean = c == '.' || c == '\u3002' || c == '\uFF0E' || c == '\uFF61'
+
+  private fun findDot(
+    value: String,
+    startIndex: Int,
+  ): Int {
+    for (i in startIndex until value.length) {
+      if (isLabelSeparator(value[i])) {
+        return i
+      }
+    }
+    return value.length
+  }
+}

src/commonMain/kotlin/io/github/optimumcode/json/schema/internal/hostname/Normalizer.kt

@@ -0,0 +1,10 @@
+package io.github.optimumcode.json.schema.internal.hostname
+
+import doist.x.normalize.Form
+import doist.x.normalize.normalize
+
+internal object Normalizer {
+  fun isNormalized(label: String): Boolean {
+    return label.normalize(Form.NFC) == label
+  }
+}

src/commonMain/kotlin/io/github/optimumcode/json/schema/internal/hostname/Punycode.kt

@@ -10,8 +10,8 @@ package io.github.optimumcode.json.schema.internal.hostname
  */
 @Suppress("detekt:MagicNumber")
 internal object Punycode {
-  private const val PREFIX_STRING = "xn--"
-  private const val PREFIX_SIZE = PREFIX_STRING.length
+  const val PREFIX_STRING = "xn--"
+  const val PREFIX_SIZE = PREFIX_STRING.length
   private const val MIN_SUPPLEMENTARY_CODE_POINT = 0x010000
   private const val MAX_CODE_POINT = 0x10FFFF
 

test-suites/src/commonTest/kotlin/io/github/optimumcode/json/schema/suite/AbstractSchemaTestSuite.kt

@@ -48,7 +48,6 @@ internal val COMMON_FORMAT_FILTER =
       mapOf(
         "email" to emptySet(),
         "idn-email" to emptySet(),
-        "idn-hostname" to emptySet(),
         "iri" to emptySet(),
         "iri-reference" to emptySet(),
         "regex" to emptySet(),