feat(isc): strip comments in sub rules, add space/non_boundary primitives

Codemod: strip # comments from sub rule lines BEFORE tokenizing. Prevents
comment text like '# comment with after keyword' from being parsed as a
real constraint.

Grammar: accept 'space' and 'non_boundary' as zero-width primitives.

Verification: 160/289 maps equivalent (55%).

Author: ronaldtse

exe/codemod-imp-to-isc

@@ -497,8 +497,11 @@ module Interscript
         # Drop the trailing newline
         line = line.chomp
 
+        # Strip comments (# ... to end of line) but only when # is at start of
+        # token (not inside a string). Walk char by char.
+        line = strip_comments(line)
+
         # Split into tokens: handle hash rockets, commas, parens, strings.
-        # We do this by walking the string with a simple state machine.
         tokens = []
         current = +""
         in_string = nil
@@ -522,31 +525,45 @@ module Interscript
           elsif paren_depth.zero? && (c == "," || (c == "=" && line[_i + 1] == ">"))
             tokens << current.strip
             current = +""
-            # Skip the comma or `=>`
-            if c == "="
-              @scanner.unscan if false # can't unscan, line already consumed
-            end
           else
             current << c
           end
         end
         tokens << current.strip unless current.strip.empty?
 
-        # Drop hash rocket tokens (already handled above by treating `=>` like `,`)
         tokens = tokens.reject { |t| t == "=>" }
 
-        # First token = from, second = to, rest = constraints
         from_expr = normalize_expr(tokens.shift.to_s)
         to_expr = normalize_expr(tokens.shift.to_s)
         constraints_str = tokens.join(" ")
 
-        # Strip the `before:` etc colon (the codemod dropped these elsewhere,
-        # but here we want to normalize: `before: X` -> `before X`)
         constraints_str = constraints_str.gsub(/(before|after|not_before|not_after)\s*:/, '\1')
 
         [from_expr, to_expr, constraints_str]
       end
 
+      # Remove `# ...` comments from a line, respecting quoted strings.
+      def strip_comments(line)
+        result = +""
+        in_string = nil
+        line.each_char do |c|
+          if in_string
+            result << c
+            if c == in_string && result[-2] != "\\"
+              in_string = nil
+            end
+          elsif c == '"' || c == "'"
+            in_string = c
+            result << c
+          elsif c == "#"
+            break
+          else
+            result << c
+          end
+        end
+        result
+      end
+
       # A "single atom" expression is one quoted string, `none`, `boundary`,
       # `line_start`, `line_end`, `word_boundary`, or a bare alias identifier.
       # Anything with `+`, `any(`, `capture(`, `maybe(`, or concatenation is

lib/interscript/isc/grammar/concerns/items.rb

@@ -26,7 +26,9 @@ module Items
               str("boundary") |
               str("line_start") |
               str("line_end") |
-              str("word_boundary")
+              str("word_boundary") |
+              str("space") |
+              str("non_boundary")
             ).as(:primitive)
           end