Merge pull request #106 from nberth/continue-ignoring-words-in-right-…

…margin

Bugfix: missing semantic tokens for words on the right of separators in the right margin

src/lsp/cobol_preproc/src_lexer.mll

@@ -515,11 +515,15 @@ and free_nominal state
 
 and gobble_line state
   = parse
-  | (nnl* newline)
+  | (nnl+)
+      {
+        gobble_line (Src_lexing.skip state lexbuf) lexbuf
+      }
+  | newline
       {
         Src_lexing.new_line state lexbuf
       }
-  | (nnl* eof)
+  | eof
       {
         Src_lexing.(flush @@ eof state lexbuf)
       }

src/lsp/cobol_preproc/src_lexing.ml

@@ -131,6 +131,11 @@ let raw_loc ~start_pos ~end_pos { newline; config = { source_format; _ }; _ } =
 let ignore_lexloc ~start_pos ~end_pos state =
   { state with ignored = (start_pos, end_pos) :: state.ignored }
 
+let skip state lexbuf =
+  ignore_lexloc state
+    ~start_pos:(Lexing.lexeme_start_p lexbuf)
+    ~end_pos:(Lexing.lexeme_end_p lexbuf)
+
 let emit prod ({ pseudotext; cdir_seen; _ } as state) =
   match pseudotext with
   | None ->

src/lsp/cobol_preproc/src_lexing.mli

@@ -26,6 +26,7 @@ val flush: 'a state -> 'a state * Text.text
 val flush_continued: ?force:bool -> 'a state -> 'a state
 val eof: 'a state -> Lexing.lexbuf -> 'a state
 val new_line: 'a state -> Lexing.lexbuf -> 'a state * Text.text
+val skip: 'a state -> Lexing.lexbuf -> 'a state
 
 val comment
   : ?marker:string