Refine catcodes returned by v-spec args

latex3 · Jan 11, 2025 · 6dba2e8 · 6dba2e8
1 parent 0071423
commit 6dba2e8
Show file tree

Hide file tree

Showing 47 changed files with 304 additions and 80 deletions.
diff --git a/base/changes.txt b/base/changes.txt
@@ -6,10 +6,11 @@ to completeness or accuracy and it contains some references to files that are
 not part of the distribution.
 ================================================================================
 
-2025-01-08  Joseph Wright  <[email protected]>
+2025-01-11  Joseph Wright  <[email protected]>
 
 	* ltcmd.dtx:
 	Correct catcode of space/tab in v-type arg
+	Adjust handling of catcodes for letters in v-type arg
 
 2025-01-03  Frank Mittelbach  <[email protected]>
 

diff --git a/base/doc/ltnews41.tex b/base/doc/ltnews41.tex
@@ -278,6 +278,13 @@ \subsection{Tab character as a special}
 be used in for example a \texttt{v}~specification document command without
 additional steps.
 
+\subsection{Refinement of \texttt{v}~specification category codes}
+
+Work on verbatim argument handling has highlighted that storing
+all characters as \enquote{other} (category code~12) when using a
+\texttt{v}~specification in \pkg{ltcmd} was problematic. We have now
+revised this to capture letters with their original cateogry code.
+
 \subsection{Logging text command and symbol declarations}
 
 For thirty years the documentation claimed that \cs{DeclareTextSymbol},

diff --git a/base/doc/usrguide.tex b/base/doc/usrguide.tex
@@ -43,7 +43,7 @@
     \texttt{usrguide.tex} for full details.}%
 }
 
-\date{2024-11-17}
+\date{2025-01-11}
 
 \NewDocumentCommand\cs{m}{\texttt{\textbackslash\detokenize{#1}}}
 \NewDocumentCommand\marg{m}{\arg{#1}}
@@ -837,10 +837,12 @@ \subsection{Using the verbatim argument types}
 
 Some additional details that may be useful for those with more \TeX{}
 knowledge: do not worry if this does not make sense to you! Spaces and tabs are
-stored as active characters. In Unicode engines, all other characters are of
-type \enquote{other}. In $8$-bit engines, the ASCII characters other than tab
-and space are of type \enquote{other}, and non-ASCII characters are active. As
-such, token-based comparisons are likely to fail unless set up properly.
+stored as active characters. In $8$-bit engines and non-ASCII characters are
+active, whilst other than the letters a--zA--Z, ASCII characters are
+\enquote{other}. In Unicode engines, non-ASCII codepoints will be either
+letters or \enquote{other}, based on the standard \LaTeX{} settings derived
+from Unicode data. As such, token-based comparisons are likely to fail unless
+set up properly.
 
 \subsection{Typesetting verbatim-like material}
 

diff --git a/base/ltcmd.dtx b/base/ltcmd.dtx
@@ -35,7 +35,7 @@
 %
 %    \begin{macrocode}
 \def\ltcmdversion{v1.2g}
-\def\ltcmddate{2025-01-08}
+\def\ltcmddate{2025-01-11}
 %    \end{macrocode}
 %
 %<*driver>
@@ -3663,7 +3663,7 @@
 % \end{macro}
 %
 % \begin{macro}{\@@_grab_v_aux_catcodes:}
-% \changes{v1.2g}{2025-01-08}{Store spaces and tabs as active chars}
+% \changes{v1.2g}{2025-01-11}{Store spaces and tabs as active chars}
 % \begin{macro}{\@@_grab_v_aux_abort:n}
 %   The approach for short verbatim arguments is to make the end-line
 %   character a macro parameter character: this is forbidden by the
@@ -3720,26 +3720,38 @@
 %
 % \begin{macro}{\@@_grab_v_aux_put:N}
 % \changes{v1.2d}{2024/03/21}{Collect \cs{endlinechar} as \cs{obeyedline}}
-%   Storing one token in the collected argument. Most tokens are
-%   converted to category code $12$, with the exception of active
-%   characters, and spaces (not sure what should be done for those).
+% \changes{v1.2g}{2025-01-11}{Simplify catcode handling}
+%   Storing one token in the collected argument: everything as-is except
+%   for end-of-lines, with \cs{exp_not:N} to handle actives.
 %    \begin{macrocode}
-%<latexrelease>\IncludeInRelease{2024/06/01}{\@@_grab_v_aux_put:N}%
-%<latexrelease>  {Endlines~as~\obeyedline}
+%<latexrelease>\IncludeInRelease{2025-06-01}{\@@_grab_v_aux_put:N}%
+%<latexrelease>  {Use~more~std~catcodes}
 \cs_new_protected:Npn \@@_grab_v_aux_put:N #1
   {
     \tl_put_right:Nx \l_@@_v_arg_tl
       {
-        \token_if_active:NTF #1
+        \int_compare:nNnTF {`#1} = \tex_endlinechar:D
+          { \exp_not:N \obeyedline }
           { \exp_not:N #1 }
-          {
-            \int_compare:nNnTF {`#1} = \tex_endlinechar:D
-              { \exp_not:N \obeyedline }
-              { \token_to_str:N #1 }
-          }
       }
   }
 %<latexrelease>\EndIncludeInRelease
+%<latexrelease>\IncludeInRelease{2024/06/01}{\@@_grab_v_aux_put:N}%
+%<latexrelease>  {Endlines~as~\obeyedline}
+%<latexrelease>\cs_new_protected:Npn \@@_grab_v_aux_put:N #1
+%<latexrelease>  {
+%<latexrelease>    \tl_put_right:Nx \l_@@_v_arg_tl
+%<latexrelease>      {
+%<latexrelease>        \token_if_active:NTF #1
+%<latexrelease>          { \exp_not:N #1 }
+%<latexrelease>          {
+%<latexrelease>            \int_compare:nNnTF {`#1} = \tex_endlinechar:D
+%<latexrelease>              { \exp_not:N \obeyedline }
+%<latexrelease>              { \token_to_str:N #1 }
+%<latexrelease>          }
+%<latexrelease>      }
+%<latexrelease>  }
+%<latexrelease>\EndIncludeInRelease
 %<latexrelease>\IncludeInRelease{2020/10/01}{\@@_grab_v_aux_put:N}%
 %<latexrelease>  {Endlines~as~\obeyedline}
 %<latexrelease>\cs_new_protected:Npn \@@_grab_v_aux_put:N #1

diff --git a/base/testfiles-ltcmd/github-0876.tlg b/base/testfiles-ltcmd/github-0876.tlg
@@ -1,48 +1,48 @@
 This is a generated file for the LaTeX2e validation system.
 Don't change this file in any respect.
 The token list contains the tokens:
->  b (the character b)
->  a (the character a)
->  r (the character r)
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r)
 >    (active character=macro:-> )
->  b (the character b)
->  a (the character a)
->  r (the character r).
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r).
 <recently read> }
 l. ...bar+
          ^^M
 The token list contains the tokens:
 >    (active character=macro:-> )
->  b (the character b)
->  a (the character a)
->  r (the character r)
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r)
 >  \obeyedline (control sequence=\protected macro:->\par )
->  b (the character b)
->  a (the character a)
->  r (the character r).
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r).
 <recently read> }
 l. ...bar+
          ^^M
 The token list contains the tokens:
->  b (the character b)
->  a (the character a)
->  r (the character r)
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r)
 >  \obeyedline (control sequence=\protected macro:->\par )
->  b (the character b)
->  a (the character a)
->  r (the character r).
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r).
 <recently read> }
 l. ...bar+
          ^^M
 The token list contains the tokens:
 >  \obeyedline (control sequence=\protected macro:->\par )
->  b (the character b)
->  a (the character a)
->  r (the character r)
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r)
 >  \obeyedline (control sequence=\protected macro:->\par )
->  b (the character b)
->  a (the character a)
->  r (the character r).
+>  b (the letter b)
+>  a (the letter a)
+>  r (the letter r).
 <recently read> }
 l. ...bar+
          ^^M
diff --git a/base/testfiles-ltcmd/ltcmd003.lvt b/base/testfiles-ltcmd/ltcmd003.lvt
@@ -24,16 +24,10 @@
         I~got~'\token_to_str:N\foo
         \IfBooleanT #1 {*}
         \IfNoValueF {#2} {[\tl_to_str:n{#2}]}
-        -#3-
+        -[See~below]-
         \tl_to_str:n { {#4} }
       }
-    \IfNoValueF {#3}
-      {
-        \exp_args:No \tl_if_eq:nnF
-          { \tl_to_str:n {#3} }
-          {#3}
-          {\ERROR}
-      }
+    \tl_analysis_show:n {#3}
   }
 \foo+\foo+{}
 \foo+% ^^+^^+