Refactor quick ascii character

Sources/_StringProcessing/Engine/InstPayload.swift

@@ -378,7 +378,7 @@ extension Instruction.Payload {
 struct QuantifyPayload: RawRepresentable {
   let rawValue: UInt64
   enum PayloadType: UInt64 {
-    case bitset = 0
+    case asciiBitset = 0
     case asciiChar = 1
     case any = 2
     case builtin = 4
@@ -448,7 +448,7 @@ struct QuantifyPayload: RawRepresentable {
   ) {
     assert(bitset.bits <= _payloadMask)
     self.rawValue = bitset.bits
-      + QuantifyPayload.packInfoValues(kind, minTrips, maxExtraTrips, .bitset, isScalarSemantics: isScalarSemantics)
+      + QuantifyPayload.packInfoValues(kind, minTrips, maxExtraTrips, .asciiBitset, isScalarSemantics: isScalarSemantics)
   }
 
   init(

Sources/_StringProcessing/Engine/MEBuiltins.swift

@@ -191,8 +191,10 @@ extension String {
     isScalarSemantics: Bool
   ) -> QuickResult<String.Index?> {
     assert(currentPosition < end)
-    guard let (asciiValue, next, isCRLF) = _quickASCIICharacter(
-      at: currentPosition, limitedBy: end
+    guard let (asciiValue, isCRLF: isCRLF, next) = _quickASCIICharacter(
+      at: currentPosition,
+      limitedBy: end,
+      isScalarSemantics: isScalarSemantics
     ) else {
       return .unknown
     }

Sources/_StringProcessing/Engine/MEQuantify.swift

@@ -3,8 +3,8 @@ extension Processor {
     let isScalarSemantics = payload.isScalarSemantics
 
     switch payload.type {
-    case .bitset:
-      return input.matchBitset(
+    case .asciiBitset:
+      return input.matchASCIIBitset(
         registers[payload.bitset],
         at: currentPosition,
         limitedBy: end,

Sources/_StringProcessing/Engine/Processor.swift

@@ -291,7 +291,7 @@ extension Processor {
     _ bitset: DSLTree.CustomCharacterClass.AsciiBitset,
     isScalarSemantics: Bool
   ) -> Bool {
-    guard let next = input.matchBitset(
+    guard let next = input.matchASCIIBitset(
       bitset,
       at: currentPosition,
       limitedBy: end,
@@ -723,22 +723,46 @@ extension String {
     return idx
   }
 
-  func matchBitset(
+  func matchASCIIBitset(
     _ bitset: DSLTree.CustomCharacterClass.AsciiBitset,
     at pos: Index,
     limitedBy end: Index,
     isScalarSemantics: Bool
   ) -> Index? {
-    // TODO: extremely quick-check-able
-    // TODO: can be sped up with string internals
-    if isScalarSemantics {
-      guard pos < end else { return nil }
-      guard bitset.matches(unicodeScalars[pos]) else { return nil }
-      return unicodeScalars.index(after: pos)
-    } else {
-      guard let (char, next) = characterAndEnd(at: pos, limitedBy: end),
-            bitset.matches(char) else { return nil }
-      return next
+
+    // FIXME: Inversion should be tracked and handled in only one place.
+    // That is, we should probably store it as a bit in the instruction, so that
+    // bitset matching and bitset inversion is bit-based rather that semantically
+    // inverting the notion of a match or not. As-is, we need to track both
+    // meanings in some code paths.
+    let isInverted = bitset.isInverted
+
+    guard let (asciiByte, isCRLF: isCRLF, next) = _quickASCIICharacter(
+      at: pos,
+      limitedBy: end,
+      isScalarSemantics: isScalarSemantics
+    ) else {
+      if isScalarSemantics {
+        guard pos < end else { return nil }
+        guard bitset.matches(unicodeScalars[pos]) else { return nil }
+        return unicodeScalars.index(after: pos)
+      } else {
+        guard let (char, next) = characterAndEnd(at: pos, limitedBy: end),
+              bitset.matches(char) else { return nil }
+        return next
+      }
     }
+
+    guard bitset.matches(asciiByte) else {
+      // FIXME: check inversion here after refactored out of bitset
+      return nil
+    }
+
+    // CR-LF should only match `[\r]` in scalar semantic mode or if inverted
+    if isCRLF && !isScalarSemantics && !isInverted {
+      return nil
+    }
+
+    return next
   }
 }

Sources/_StringProcessing/Unicode/ASCII.swift

@@ -81,13 +81,22 @@ extension UInt8 {
 }
 
 extension String {
-  /// TODO: better to take isScalarSemantics parameter, we can return more results
-  /// and we can give the right `next` index, not requiring the caller to re-adjust it
-  /// TODO: detailed description of nuanced semantics
+  ///
+  /// If the position in the input is not definitely a full-ASCII character (uses sub-`0x300` quick check
+  /// on next byte), return `nil.`
+  ///
+  /// Otherwise, returns:
+  ///   1. The first ASCII byte for a character (or scalar if `isScalarSemantics`)
+  ///   2. Whether the ASCII Character is CR-LF
+  ///   3. The index for the end of that particular ASCII character:
+  ///     If the character is not CR-LF, the index of the next byte
+  ///     if `isScalarSemantics` is false and the ASCII character is CR-LF, the index after the CR-LF sequence.
+  ///
   func _quickASCIICharacter(
     at idx: Index,
-    limitedBy end: Index
-  ) -> (first: UInt8, next: Index, crLF: Bool)? {
+    limitedBy end: Index,
+    isScalarSemantics: Bool
+  ) -> (firstASCIIByte: UInt8, isCRLF: Bool, asciiCharacterEnd: Index)? {
     // TODO: fastUTF8 version
     assert(String.Index(idx, within: unicodeScalars) != nil)
     assert(idx <= end)
@@ -101,25 +110,25 @@ extension String {
       return nil
     }
 
-    var next = utf8.index(after: idx)
-    if next == end {
-      return (first: base, next: next, crLF: false)
+    let byteEnd = utf8.index(after: idx)
+    if isScalarSemantics || byteEnd == end {
+      return (firstASCIIByte: base, isCRLF: false, asciiCharacterEnd: byteEnd)
     }
 
-    let tail = utf8[next]
+    let tail = utf8[byteEnd]
     guard tail._isSub300StartingByte else { return nil }
 
     // Handle CR-LF:
     if base == ._carriageReturn && tail == ._lineFeed {
-      utf8.formIndex(after: &next)
-      guard next == end || utf8[next]._isSub300StartingByte else {
+      let crLFEnd = utf8.index(after: byteEnd)
+      guard crLFEnd == end || utf8[crLFEnd]._isSub300StartingByte else {
         return nil
       }
-      return (first: base, next: next, crLF: true)
+      return (firstASCIIByte: base, isCRLF: true, asciiCharacterEnd: crLFEnd)
     }
 
     assert(self[idx].isASCII && self[idx] != "\r\n")
-    return (first: base, next: next, crLF: false)
+    return (firstASCIIByte: base, isCRLF: false, asciiCharacterEnd: byteEnd)
   }
 
   func _quickMatch(
@@ -128,44 +137,47 @@ extension String {
     limitedBy end: Index,
     isScalarSemantics: Bool
   ) -> (next: Index, matchResult: Bool)? {
+    // Don't use scalar semantics in this quick path for anyGrapheme cluster or
+    // newline sequences, which are not scalar character classes.
+    let useScalarSemantics = isScalarSemantics && cc != .anyGrapheme && cc != .newlineSequence
     /// ASCII fast-paths
-    guard let (asciiValue, next, isCRLF) = _quickASCIICharacter(
-      at: idx, limitedBy: end
+    guard let (asciiValue, isCRLF: isCRLF, charEnd) = _quickASCIICharacter(
+      at: idx,
+      limitedBy: end,
+      isScalarSemantics: useScalarSemantics
     ) else {
       return nil
     }
 
     // TODO: bitvectors
     switch cc {
-    case .any, .anyGrapheme:
-      return (next, true)
+    case .any:
+      return (charEnd, true)
+
+    case .anyGrapheme:
+      // _quickASCIICharacter call handled CR-LF for us
+      _ = isCRLF
+      return (charEnd, true)
 
     case .digit:
-      return (next, asciiValue._asciiIsDigit)
+      return (charEnd, asciiValue._asciiIsDigit)
 
     case .horizontalWhitespace:
-      return (next, asciiValue._asciiIsHorizontalWhitespace)
-
-    case .verticalWhitespace, .newlineSequence:
-      if asciiValue._asciiIsVerticalWhitespace {
-        if isScalarSemantics && isCRLF && cc == .verticalWhitespace {
-          return (utf8.index(before: next), true)
-        }
-        return (next, true)
-      }
-      return (next, false)
+      return (charEnd, asciiValue._asciiIsHorizontalWhitespace)
+
+    case .verticalWhitespace:
+      return (charEnd, asciiValue._asciiIsVerticalWhitespace)
+
+    case .newlineSequence:
+      // _quickASCIICharacter call handled CR-LF for us
+      _ = isCRLF
+      return (charEnd, asciiValue._asciiIsVerticalWhitespace)
 
     case .whitespace:
-      if asciiValue._asciiIsWhitespace {
-        if isScalarSemantics && isCRLF {
-          return (utf8.index(before: next), true)
-        }
-        return (next, true)
-      }
-      return (next, false)
+      return (charEnd, asciiValue._asciiIsWhitespace)
 
     case .word:
-      return (next, asciiValue._asciiIsWord)
+      return (charEnd, asciiValue._asciiIsWord)
     }
   }
 

Sources/_StringProcessing/Utility/AsciiBitset.swift

@@ -1,3 +1,4 @@
+// TODO: Probably refactor out of DSLTree
 extension DSLTree.CustomCharacterClass {
   internal struct AsciiBitset {
     let isInverted: Bool
@@ -49,18 +50,23 @@ extension DSLTree.CustomCharacterClass {
       }
     }
 
-    private func matches(_ val: UInt8) -> Bool {
+    private func _matchesWithoutInversionCheck(_ val: UInt8) -> Bool {
       if val < 64 {
         return (a >> val) & 1 == 1
       } else {
         return (b >> (val - 64)) & 1 == 1
       }
     }
 
+    internal func matches(_ byte: UInt8) -> Bool {
+      guard byte < 128 else { return isInverted }
+      return _matchesWithoutInversionCheck(byte) == !isInverted
+    }
+
     internal func matches(_ char: Character) -> Bool {
       let matched: Bool
       if let val = char._singleScalarAsciiValue {
-        matched = matches(val)
+        matched = _matchesWithoutInversionCheck(val)
       } else {
         matched = false
       }
@@ -75,7 +81,7 @@ extension DSLTree.CustomCharacterClass {
       let matched: Bool
       if scalar.isASCII {
         let val = UInt8(ascii: scalar)
-        matched = matches(val)
+        matched = _matchesWithoutInversionCheck(val)
       } else {
         matched = false
       }