390 ignore linemarkers

doc/source/fparser2.rst

@@ -552,6 +552,12 @@ backslash character `\\` at the end of the line.
 
 __ http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf#page=157
 
+Added is the support for compiler linemarkers, i.e. lines in the format
+``# line-number "filename"``, which indicates for a compiler the line number
+and filename that the next line came from. While technically not a preprocessor
+directive, these statements follow a very similar syntax so their handling
+is combined with the preprocessor handling.
+
 The implementation of directives is in the C99Preprocessor.py `file`__
 with support for the following::
 
@@ -568,6 +574,7 @@ with support for the following::
   #error
   #warning
   #
+  # line-number "filename"
 
 __ https://github.com/stfc/fparser/blob/master/src/fparser/two/C99Preprocessor.py
 

src/fparser/two/C99Preprocessor.py

@@ -32,14 +32,19 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""C99 Preprocessor Syntax Rules."""
+"""C99 Preprocessor Syntax Rules. It also supports linemarker statements
+(which are technically not preprocessor directives, but are very close
+in their syntax, i.e. starting with `#`)
+
+"""
 
 # Author: Balthasar Reuter <balthasar.reuter@ecmwf.int>
 # Based on previous work by Martin Schlipf (https://github.com/martin-schlipf)
 # First version created: Jan 2020
 
 import re
 import sys
+from typing import Optional, Union
 
 from fparser.common.readfortran import FortranReaderBase, CppDirective
 from fparser.two import pattern_tools as pattern
@@ -57,6 +62,7 @@
     "Cpp_Macro_Stmt",
     "Cpp_Undef_Stmt",
     "Cpp_Line_Stmt",
+    "Cpp_Linemarker_Stmt",
     "Cpp_Error_Stmt",
     "Cpp_Warning_Stmt",
     "Cpp_Null_Stmt",
@@ -649,6 +655,64 @@ def tostr(self):
         return "{0} {1}".format(*self.items)
 
 
+class Cpp_Linemarker_Stmt(WORDClsBase):  # Linemarker
+    """
+    This class represents a Linemarker. A linemarker indicates the
+    line number and file name the following line is coming from (e.g.
+    if a file has been inlined, this will allow the compiler to correctly
+    indicate the original source line). While linemarkers are technically
+    not preprocessor directives, their syntax is very similar, so they are
+    handled here.
+
+    linemarker-stmt is # digit-sequence "s-char-sequence" [digit ...]
+    """
+
+    subclass_names = []
+    use_names = ["Cpp_Pp_Tokens"]
+
+    # The match method will check that it is a valid linemarker, i.e.
+    # it has a line number, and file name in double quotes. Setting value
+    # to None means that the pattern matching will return the matched
+    # string (i.e. `# linenumber "filename"`), any following flags will
+    # be stored as items of type Cpp_Pp_Tokens.
+    _pattern = pattern.Pattern("<linemarker>", r"^\s*#\s+\d+\s+\".*\".*$", value=None)
+
+    @staticmethod
+    def match(
+        string: Union[str, FortranReaderBase],
+    ) -> Optional[tuple[str, "Cpp_Linemarker_Stmt"]]:
+        """Implements the matching for a linemarker.
+        The optional flag (digits) allowed after the file name are not matched
+        any further but simply kept as a string.
+
+        :param string: the string to match with as a line statement.
+
+        :return: an instance of Cpp_Linemarker_Stmt or `None` if there is no
+            match.
+
+        """
+        if not string:
+            return None
+
+        return WORDClsBase.match(
+            Cpp_Linemarker_Stmt._pattern,
+            Cpp_Pp_Tokens,
+            string,
+            colons=False,
+            require_cls=False,
+        )
+
+    def tostr(self) -> str:
+        """
+        Returns the line marker as string. Note that fparser accepts
+        spaces before the `#`, but it should remove the spaces, hence
+        we lstrip the result
+
+        :return: this linemarker as a string.
+        """
+        return self.items[0].lstrip()
+
+
 class Cpp_Error_Stmt(WORDClsBase):  # 6.10.5 Error directive
     """
     C99 6.10.5 Error directive

src/fparser/two/tests/test_c99preprocessor.py

@@ -57,12 +57,14 @@
     Cpp_Macro_Identifier_List,
     Cpp_Undef_Stmt,
     Cpp_Line_Stmt,
+    Cpp_Linemarker_Stmt,
     Cpp_Error_Stmt,
     Cpp_Warning_Stmt,
     Cpp_Null_Stmt,
     Cpp_Pp_Tokens,
 )
-from fparser.two.utils import NoMatchError
+from fparser.two.Fortran2003 import Program
+from fparser.two.utils import NoMatchError, walk
 from fparser.api import get_reader
 
 
@@ -366,7 +368,8 @@ def test_macro_stmt_with_whitespace(line, ref):
         "#def",
         "#defnie",
         "#definex",
-        "#define 2a" "#define fail(...,test) test",
+        "#define 2a",
+        "#define fail(...,test) test",
         "#define",
         "#define fail(...,...)",
     ],
@@ -451,6 +454,30 @@ def test_incorrect_line_stmt(line):
     assert "Cpp_Line_Stmt: '{0}'".format(line) in str(excinfo.value)
 
 
+@pytest.mark.usefixtures("f2003_create")
+@pytest.mark.parametrize(
+    "line, ref",
+    [
+        ('# 123 "file"', '# 123 "file"'),
+        ('  #     123 "file"', '#     123 "file"'),
+        ('# 123 "file" 1 3', '# 123 "file" 1 3'),
+    ],
+)
+def test_linemarker(line, ref):
+    """Test that #line is recognized"""
+    result = Cpp_Linemarker_Stmt(line)
+    assert str(result) == ref
+
+
+@pytest.mark.usefixtures("f2003_create")
+@pytest.mark.parametrize("line", ["# abc", '# "bla"', "# 123 'wrong_quotes'"])
+def test_incorrect_linemarker(line):
+    """Test that incorrectly formed #line statements raise exception"""
+    with pytest.raises(NoMatchError) as excinfo:
+        _ = Cpp_Linemarker_Stmt(line)
+    assert "Cpp_Linemarker_Stmt: '{0}'".format(line) in str(excinfo.value)
+
+
 @pytest.mark.usefixtures("f2003_create")
 @pytest.mark.parametrize("line", ["#error MSG", "  #  error  MSG  "])
 def test_error_statement_with_msg(line):
@@ -525,3 +552,42 @@ def test_incorrect_null_stmt(line):
     with pytest.raises(NoMatchError) as excinfo:
         _ = Cpp_Null_Stmt(line)
     assert "Cpp_Null_Stmt: '{0}'".format(line) in str(excinfo.value)
+
+
+@pytest.mark.usefixtures("f2003_create")
+@pytest.mark.parametrize(
+    "cpp_class, cpp_directive",
+    [
+        (Cpp_If_Stmt, "#if CONST"),
+        (Cpp_Elif_Stmt, "#elif CONST"),
+        (Cpp_Endif_Stmt, "#endif"),
+        (Cpp_Include_Stmt, '#include "test.inc"'),
+        (Cpp_Macro_Stmt, "#define a b"),
+        (Cpp_Undef_Stmt, "#undef a"),
+        (Cpp_Line_Stmt, "#line 123"),
+        (Cpp_Linemarker_Stmt, '# 123 "test.f90"'),
+        (Cpp_Error_Stmt, "#error 123"),
+        (Cpp_Warning_Stmt, "#warning 123"),
+        (Cpp_Null_Stmt, "#"),
+    ],
+)
+def test_cpp_in_fortran(cpp_class, cpp_directive):
+    """
+    Verify that all cpp directives are correctly parsed as part of
+    a real program.
+    """
+    code = f"""
+    program test
+    {cpp_directive}
+    integer a
+    a = 2
+    end program
+    """
+    reader = get_reader(code)
+
+    obj = Program(reader)
+    all_cpp_nodes = walk(obj, cpp_class)
+
+    # There must be exactly one cpp node
+    assert len(all_cpp_nodes) == 1
+    assert str(all_cpp_nodes[0]) == cpp_directive

src/fparser/two/utils.py

@@ -1792,7 +1792,7 @@ def match(keyword, cls, string, colons=False, require_cls=False):
             2-tuple containing a string matching the 'WORD' and an \
             instance of 'cls' (or None if an instance of cls is not \
             required and not provided).
-        :rtype: Optional[Tupe[Str, Optional[Cls]]]
+        :rtype: Optional[Tuple[Str, Optional[Cls]]]
 
         """
         if isinstance(keyword, (tuple, list)):
@@ -1818,7 +1818,18 @@ def match(keyword, cls, string, colons=False, require_cls=False):
             if my_match is None:
                 return None
             line = string[len(my_match.group()) :]
-            pattern_value = keyword.value
+            # Most patterns set a return value to be used, in order to remove
+            # white space (e.g. the pattern might be "^\s*(#\s*undef)\b",
+            # but the return value is `#undef`, meaning all optional white
+            # space will be removed. But in case of linemarkers, we need
+            # to match a non-constant expression (`# linenumber "filename"`).
+            # In this case, value is set to None, and we return the matched
+            # original string (i.e. the actual line number and filename
+            # specified)
+            if keyword.value:
+                pattern_value = keyword.value
+            else:
+                pattern_value = my_match.group()
 
         if not line:
             if require_cls: