diff --git a/doc/source/fparser2.rst b/doc/source/fparser2.rst index a7571db7..a827cd61 100644 --- a/doc/source/fparser2.rst +++ b/doc/source/fparser2.rst @@ -552,6 +552,12 @@ backslash character `\\` at the end of the line. __ http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf#page=157 +Added is the support for compiler linemarkers, i.e. lines in the format +``# line-number "filename"``, which indicates for a compiler the line number +and filename that the next line came from. While technically not a preprocessor +directive, these statements follow a very similar syntax so their handling +is combined with the preprocessor handling. + The implementation of directives is in the C99Preprocessor.py `file`__ with support for the following:: @@ -568,6 +574,7 @@ with support for the following:: #error #warning # + # line-number "filename" __ https://github.com/stfc/fparser/blob/master/src/fparser/two/C99Preprocessor.py diff --git a/src/fparser/two/C99Preprocessor.py b/src/fparser/two/C99Preprocessor.py index 3b42a211..d7841699 100644 --- a/src/fparser/two/C99Preprocessor.py +++ b/src/fparser/two/C99Preprocessor.py @@ -32,7 +32,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""C99 Preprocessor Syntax Rules.""" +"""C99 Preprocessor Syntax Rules. It also supports linemarker statements +(which are technically not preprocessor directives, but are very close +in their syntax, i.e. starting with `#`) + +""" # Author: Balthasar Reuter # Based on previous work by Martin Schlipf (https://github.com/martin-schlipf) @@ -40,6 +44,7 @@ import re import sys +from typing import Optional, Union from fparser.common.readfortran import FortranReaderBase, CppDirective from fparser.two import pattern_tools as pattern @@ -57,6 +62,7 @@ "Cpp_Macro_Stmt", "Cpp_Undef_Stmt", "Cpp_Line_Stmt", + "Cpp_Linemarker_Stmt", "Cpp_Error_Stmt", "Cpp_Warning_Stmt", "Cpp_Null_Stmt", @@ -649,6 +655,64 @@ def tostr(self): return "{0} {1}".format(*self.items) +class Cpp_Linemarker_Stmt(WORDClsBase): # Linemarker + """ + This class represents a Linemarker. A linemarker indicates the + line number and file name the following line is coming from (e.g. + if a file has been inlined, this will allow the compiler to correctly + indicate the original source line). While linemarkers are technically + not preprocessor directives, their syntax is very similar, so they are + handled here. + + linemarker-stmt is # digit-sequence "s-char-sequence" [digit ...] + """ + + subclass_names = [] + use_names = ["Cpp_Pp_Tokens"] + + # The match method will check that it is a valid linemarker, i.e. + # it has a line number, and file name in double quotes. Setting value + # to None means that the pattern matching will return the matched + # string (i.e. `# linenumber "filename"`), any following flags will + # be stored as items of type Cpp_Pp_Tokens. + _pattern = pattern.Pattern("", r"^\s*#\s+\d+\s+\".*\".*$", value=None) + + @staticmethod + def match( + string: Union[str, FortranReaderBase], + ) -> Optional[tuple[str, "Cpp_Linemarker_Stmt"]]: + """Implements the matching for a linemarker. + The optional flag (digits) allowed after the file name are not matched + any further but simply kept as a string. + + :param string: the string to match with as a line statement. + + :return: an instance of Cpp_Linemarker_Stmt or `None` if there is no + match. + + """ + if not string: + return None + + return WORDClsBase.match( + Cpp_Linemarker_Stmt._pattern, + Cpp_Pp_Tokens, + string, + colons=False, + require_cls=False, + ) + + def tostr(self) -> str: + """ + Returns the line marker as string. Note that fparser accepts + spaces before the `#`, but it should remove the spaces, hence + we lstrip the result + + :return: this linemarker as a string. + """ + return self.items[0].lstrip() + + class Cpp_Error_Stmt(WORDClsBase): # 6.10.5 Error directive """ C99 6.10.5 Error directive diff --git a/src/fparser/two/tests/test_c99preprocessor.py b/src/fparser/two/tests/test_c99preprocessor.py index 921796a6..37f4ef83 100644 --- a/src/fparser/two/tests/test_c99preprocessor.py +++ b/src/fparser/two/tests/test_c99preprocessor.py @@ -57,12 +57,14 @@ Cpp_Macro_Identifier_List, Cpp_Undef_Stmt, Cpp_Line_Stmt, + Cpp_Linemarker_Stmt, Cpp_Error_Stmt, Cpp_Warning_Stmt, Cpp_Null_Stmt, Cpp_Pp_Tokens, ) -from fparser.two.utils import NoMatchError +from fparser.two.Fortran2003 import Program +from fparser.two.utils import NoMatchError, walk from fparser.api import get_reader @@ -366,7 +368,8 @@ def test_macro_stmt_with_whitespace(line, ref): "#def", "#defnie", "#definex", - "#define 2a" "#define fail(...,test) test", + "#define 2a", + "#define fail(...,test) test", "#define", "#define fail(...,...)", ], @@ -451,6 +454,30 @@ def test_incorrect_line_stmt(line): assert "Cpp_Line_Stmt: '{0}'".format(line) in str(excinfo.value) +@pytest.mark.usefixtures("f2003_create") +@pytest.mark.parametrize( + "line, ref", + [ + ('# 123 "file"', '# 123 "file"'), + (' # 123 "file"', '# 123 "file"'), + ('# 123 "file" 1 3', '# 123 "file" 1 3'), + ], +) +def test_linemarker(line, ref): + """Test that #line is recognized""" + result = Cpp_Linemarker_Stmt(line) + assert str(result) == ref + + +@pytest.mark.usefixtures("f2003_create") +@pytest.mark.parametrize("line", ["# abc", '# "bla"', "# 123 'wrong_quotes'"]) +def test_incorrect_linemarker(line): + """Test that incorrectly formed #line statements raise exception""" + with pytest.raises(NoMatchError) as excinfo: + _ = Cpp_Linemarker_Stmt(line) + assert "Cpp_Linemarker_Stmt: '{0}'".format(line) in str(excinfo.value) + + @pytest.mark.usefixtures("f2003_create") @pytest.mark.parametrize("line", ["#error MSG", " # error MSG "]) def test_error_statement_with_msg(line): @@ -525,3 +552,42 @@ def test_incorrect_null_stmt(line): with pytest.raises(NoMatchError) as excinfo: _ = Cpp_Null_Stmt(line) assert "Cpp_Null_Stmt: '{0}'".format(line) in str(excinfo.value) + + +@pytest.mark.usefixtures("f2003_create") +@pytest.mark.parametrize( + "cpp_class, cpp_directive", + [ + (Cpp_If_Stmt, "#if CONST"), + (Cpp_Elif_Stmt, "#elif CONST"), + (Cpp_Endif_Stmt, "#endif"), + (Cpp_Include_Stmt, '#include "test.inc"'), + (Cpp_Macro_Stmt, "#define a b"), + (Cpp_Undef_Stmt, "#undef a"), + (Cpp_Line_Stmt, "#line 123"), + (Cpp_Linemarker_Stmt, '# 123 "test.f90"'), + (Cpp_Error_Stmt, "#error 123"), + (Cpp_Warning_Stmt, "#warning 123"), + (Cpp_Null_Stmt, "#"), + ], +) +def test_cpp_in_fortran(cpp_class, cpp_directive): + """ + Verify that all cpp directives are correctly parsed as part of + a real program. + """ + code = f""" + program test + {cpp_directive} + integer a + a = 2 + end program + """ + reader = get_reader(code) + + obj = Program(reader) + all_cpp_nodes = walk(obj, cpp_class) + + # There must be exactly one cpp node + assert len(all_cpp_nodes) == 1 + assert str(all_cpp_nodes[0]) == cpp_directive diff --git a/src/fparser/two/utils.py b/src/fparser/two/utils.py index a2f3e3b7..35d1927c 100644 --- a/src/fparser/two/utils.py +++ b/src/fparser/two/utils.py @@ -1792,7 +1792,7 @@ def match(keyword, cls, string, colons=False, require_cls=False): 2-tuple containing a string matching the 'WORD' and an \ instance of 'cls' (or None if an instance of cls is not \ required and not provided). - :rtype: Optional[Tupe[Str, Optional[Cls]]] + :rtype: Optional[Tuple[Str, Optional[Cls]]] """ if isinstance(keyword, (tuple, list)): @@ -1818,7 +1818,18 @@ def match(keyword, cls, string, colons=False, require_cls=False): if my_match is None: return None line = string[len(my_match.group()) :] - pattern_value = keyword.value + # Most patterns set a return value to be used, in order to remove + # white space (e.g. the pattern might be "^\s*(#\s*undef)\b", + # but the return value is `#undef`, meaning all optional white + # space will be removed. But in case of linemarkers, we need + # to match a non-constant expression (`# linenumber "filename"`). + # In this case, value is set to None, and we return the matched + # original string (i.e. the actual line number and filename + # specified) + if keyword.value: + pattern_value = keyword.value + else: + pattern_value = my_match.group() if not line: if require_cls: