Add name-pattern token used in prefix operators ? and ?? (#158)

* Add Names Pattern tokenization
* Start regexp testing.
* Uppercase _letter{,likes} s
* Information -> QuestionQuestion
* Make change_token_scanning_mode public

---------

Co-authored-by: Juan Mauricio Matera <matera@fisica.unlp.edu.ar>

Author: rocky

.github/workflows/mathics3-doctest.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Build Mathics3
       run: |
         # Until next Mathics3/mathics-core release is out...
-        git clone --depth 1 https://github.com/Mathics3/mathics-core.git
+        git clone --depth 1 -b add-Name-pattern-token https://github.com/Mathics3/mathics-core.git
         cd mathics-core/
         python -m pip install -e .[dev]
         cp -v ../mathics_scanner/data/boxing-characters.json mathics/data/boxing-characters.json

mathics_scanner/characters.py

@@ -123,7 +123,7 @@ def replace_box_unicode_with_ascii(input_string):
 
 
 # Character ranges of letters
-_letters: Final[str] = (
+LETTERS: Final[str] = (
     "a-zA-Z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u0103\u0106\u0107\
 \u010c-\u010f\u0112-\u0115\u011a-\u012d\u0131\u0141\u0142\u0147\u0148\
 \u0150-\u0153\u0158-\u0161\u0164\u0165\u016e-\u0171\u017d\u017e\
@@ -136,7 +136,7 @@ def replace_box_unicode_with_ascii(input_string):
 )
 
 # Character ranges of letterlikes
-_letterlikes: Final[Dict[str, str]] = NAMED_CHARACTERS_COLLECTION.get("letterlikes", {})
+LETTERLIKES: Final[Dict[str, str]] = NAMED_CHARACTERS_COLLECTION.get("letterlikes", {})
 
 # Conversion from WL to the fully qualified names
 _wl_to_ascii: Final[Dict[str, str]] = NAMED_CHARACTERS_COLLECTION.get(