Rework handling of assignment words (#176)

src/CST.mli

@@ -319,7 +319,6 @@ and word_cst = word_component list
 and word_component =
   | WordSubshell of subshell_kind * program located
   | WordName of string
-  | WordAssignmentWord of assignment_word
   | WordDoubleQuoted of word
   | WordSingleQuoted of word
   | WordTildePrefix of string

src/CSTHelpers.ml

@@ -139,8 +139,8 @@ let unName (Name s) = s
 
 let word_of_name (Name w) = Word (w, [WordName w])
 
-let word_of_assignment_word (Name n, (Word (s, _) as w)) =
-  Word (n ^ "=" ^ s, [WordAssignmentWord (Name n, w)])
+let word_of_assignment_word (Name name, Word (word_str, word_cst)) =
+  Word (name ^ "=" ^ word_str, WordLiteral (name ^ "=") :: word_cst)
 
 let string_of_word (Word (s, _)) = s
 
@@ -230,3 +230,17 @@ let io_redirect_list_of_simple_command = function
     io_redirect_list_of_cmd_suffix cmd_suffix'.value
   | SimpleCommand_CmdName _ ->
     []
+
+let merge_leading_literals =
+  let buf = Buffer.create 80 in
+  let rec extract_leading_literals = function
+    | WordLiteral lit :: rest ->
+      Buffer.add_string buf lit;
+      extract_leading_literals rest
+    | rest -> rest
+  in
+  fun word ->
+    let rest = extract_leading_literals word in
+    let lit = Buffer.contents buf in
+    Buffer.reset buf;
+    if lit = "" then word else WordLiteral lit :: rest

src/CSTHelpers.mli

@@ -71,3 +71,6 @@ val io_redirect_list_of_cmd_prefix : cmd_prefix -> io_redirect' list
 val io_redirect_list_of_cmd_suffix : cmd_suffix -> io_redirect' list
 val io_redirect_list_of_simple_command : simple_command -> io_redirect' list
 val io_redirect_list_of_redirect_list : redirect_list -> io_redirect' list
+
+(** Merges several leading [WordLiteral] into one. *)
+val merge_leading_literals : word_cst -> word_cst

src/assignment.ml

@@ -41,18 +41,36 @@ open Name
 
 *)
 
-let recognize_assignment checkpoint pretoken word_cst = FirstSuccessMonad.(
+let recognize_assignment checkpoint pretoken (Word (word_str, word_cst)) =
+  FirstSuccessMonad.(
     match word_cst with
-    | [WordAssignmentWord ((Name n) as name, w)] ->
-      if is_name n then
-        let (_, pstart, pstop) = pretoken in
-        let token = ASSIGNMENT_WORD (name, w) in
-        if accepted_token checkpoint (token, pstart, pstop) <> Wrong then
-          return token
-        else
-          fail
-      else
-        fail
-    | _ ->
-      fail
+    | WordLiteral literal :: word_cst_leftover ->
+      (
+        match String.index_opt literal '=' with
+        | None | Some 0 -> fail
+        | Some i ->
+          let name = String.sub literal 0 i in
+          if is_name name then
+            let (_, pstart, pstop) = pretoken in
+            let literal_leftover = String.sub literal (i + 1) (String.length literal - i - 1) in
+            let word_str_leftover = String.sub word_str (i + 1) (String.length word_str - i - 1) in
+            let word_cst =
+              if literal_leftover = "" then
+                word_cst_leftover
+              else
+                WordLiteral literal_leftover :: word_cst_leftover
+            in
+            (* Now that we know we're in an assignment, we know there are
+               potentially more tilde prefixes to recognize. *)
+            let word_cst = TildePrefix.recognize ~in_assignment:true word_cst in
+            let word = Word (word_str_leftover, word_cst) in
+            let token = ASSIGNMENT_WORD (Name name, word) in
+            if accepted_token checkpoint (token, pstart, pstop) <> Wrong then
+              return token
+            else
+              fail
+          else
+            fail
+      )
+    | _ -> fail
   )

src/engine.ml

@@ -336,7 +336,7 @@ module Lexer (U : sig end) : Lexer = struct
             | _ -> false
         in
         let token = FirstSuccessMonad.(
-            (recognize_assignment checkpoint p cst)
+            (recognize_assignment checkpoint p (Word (w0, cst)))
             +> (recognize_reserved_word_if_relevant
                   well_delimited_keyword checkpoint p w)
             +> return word

src/prelexerState.ml

@@ -188,7 +188,7 @@ let string_of_word (Word (s, _)) = s
 
 let string_of_attribute = function
   | NoAttribute -> ""
-  | ParameterLength -> "#" 
+  | ParameterLength -> "#"
   | UseDefaultValues (p, w) -> p ^ string_of_word w
   | AssignDefaultValues (p, w) -> p ^ string_of_word w
   | IndicateErrorifNullorUnset (p, w) -> p ^ string_of_word w
@@ -287,46 +287,6 @@ let is_assignment_mark = function
   | AssignmentMark -> true
   | _ -> false
 
-let recognize_assignment current =
-  let rhs, prefix = take_until is_assignment_mark (buffer current) in
-  if prefix = buffer current then (
-    current
-  ) else
-    let buffer = AtomBuffer.make (rhs @ List.tl prefix) in
-    let current' = { current with buffer } in
-    match prefix with
-    | AssignmentMark :: WordComponent (s, _) :: prefix ->
-      assert (s.[String.length s - 1] = '='); (* By after_equal unique call. *)
-      (* [s] is a valid name. We have an assignment here. *)
-      let lhs = try String.(sub s 0 (length s - 1)) with _ -> assert false in
-
-      (* FIXME: The following check could be done directly with
-         ocamllex rules, right?*)
-
-      if Name.is_name lhs then (
-        let rhs_string = contents_of_atom_list rhs in
-        let crhs = components_of_atom_list rhs in
-        let cst =  WordComponent (
-            s ^ rhs_string,
-            WordAssignmentWord (Name lhs, Word (rhs_string, crhs)))
-        in
-        let buffer = AtomBuffer.make (cst :: prefix) in
-        { current with buffer }
-      ) else
-         (*
-            If [lhs] is not a name, then the corresponding word
-            literal must be merged with the preceding one, if it exists.
-          *) (
-        begin match List.rev rhs with
-          | WordComponent (s_rhs, WordLiteral s_rhs') :: rev_rhs ->
-            let word = WordComponent (s ^ s_rhs, WordLiteral (s ^ s_rhs')) in
-            let buffer = AtomBuffer.make (List.rev rev_rhs @ word :: prefix) in
-            { current with buffer }
-          | _ ->
-            current'
-        end)
-    | _ -> current'
-
 (** [(return ?with_newline lexbuf current tokens)] returns a list of
     pretokens consisting of, in that order:
 
@@ -349,8 +309,6 @@ let return ?(with_newline=false) lexbuf (current : prelexer_state) tokens =
     not (List.exists (function (Pretoken.PreWord _)->true |_-> false) tokens)
   );
 
-  let current = recognize_assignment current in
-
   let flush_word b =
     let buf = Buffer.create 13 in
     List.iter (function WordComponent (s, _) -> Buffer.add_string buf s
@@ -408,7 +366,9 @@ let return ?(with_newline=false) lexbuf (current : prelexer_state) tokens =
             | QuotingMark _ -> []
           ) (buffer current)))
       in
-      let csts = TildePrefix.recognize csts in
+      (* At this point, we cannot say whether this will be an assignment word,
+         so we do a minimal tilde prefixes recognition. *)
+      let csts = TildePrefix.recognize ~in_assignment:false csts in
       [Pretoken.PreWord (w, csts)]
   in
   let tokens = if with_newline then tokens @ [Pretoken.NEWLINE] else tokens in

src/tildePrefix.ml

@@ -61,25 +61,10 @@ let extract_tilde_prefix_from_literal (literal : string) : word_cst =
     let (first, rest) = ExtPervasives.string_split i literal in
     [WordTildePrefix (strip_tilde first); WordLiteral rest]
 
-(** Merges several leading [WordLiteral] into one. *)
-let merge_leading_literals : word_cst -> word_cst =
-  let buf = Buffer.create 80 in
-  let rec extract_leading_literals = function
-    | WordLiteral lit :: rest ->
-      Buffer.add_string buf lit;
-      extract_leading_literals rest
-    | rest -> rest
-  in
-  fun word ->
-    let rest = extract_leading_literals word in
-    let lit = Buffer.contents buf in
-    Buffer.reset buf;
-    if lit = "" then word else WordLiteral lit :: rest
-
 (** Extracts the tilde-prefix at the beginning of the given word CST if there is
     one. Otherwise, returns the word as-is. *)
 let extract_tilde_prefix_from_word_if_present (word : word_cst) : word_cst =
-  match merge_leading_literals word with
+  match CSTHelpers.merge_leading_literals word with
   | WordLiteral literal :: word when starts_with_tilde literal ->
     extract_tilde_prefix_from_literal literal @ word
   | word -> word
@@ -136,12 +121,10 @@ let rec concat_words_with_colon (words : word_cst list) : word_cst =
 (** Recognises tilde prefixes in a word, that is recognises eg. [WordLiteral
     "~foo"] and replaces it by [WordTildePrefix "foo"] when in the right
     position. *)
-let recognize (word : word_cst) =
-  match word with
-  | [WordAssignmentWord (name, Word (s, word))] ->
+let recognize ~in_assignment (word : word_cst) =
+  if in_assignment then
     let words = split_word_on_colon word in
     let words = List.map extract_tilde_prefix_from_word_if_present words in
-    let word = concat_words_with_colon words in
-    [WordAssignmentWord (name, Word (s, word))]
-  | _ ->
+    concat_words_with_colon words
+  else
     extract_tilde_prefix_from_word_if_present word

tests/golden/good/2.3-token-recognition/2.3.1-alias-substitution/alias-alias.t/expected.json

@@ -43,23 +43,8 @@
                           "rename=alias",
                           [
                             [
-                              "WordAssignmentWord",
-                              [
-                                [
-                                  "Name",
-                                  "rename"
-                                ],
-                                [
-                                  "Word",
-                                  "alias",
-                                  [
-                                    [
-                                      "WordLiteral",
-                                      "alias"
-                                    ]
-                                  ]
-                                ]
-                              ]
+                              "WordLiteral",
+                              "rename=alias"
                             ]
                           ]
                         ]
@@ -109,23 +94,8 @@
                         "time=date",
                         [
                           [
-                            "WordAssignmentWord",
-                            [
-                              [
-                                "Name",
-                                "time"
-                              ],
-                              [
-                                "Word",
-                                "date",
-                                [
-                                  [
-                                    "WordLiteral",
-                                    "date"
-                                  ]
-                                ]
-                              ]
-                            ]
+                            "WordLiteral",
+                            "time=date"
                           ]
                         ]
                       ]

tests/golden/good/2.3-token-recognition/2.3.1-alias-substitution/alias-commands-words-only.t/expected.json

@@ -41,23 +41,8 @@
                         "show=echo",
                         [
                           [
-                            "WordAssignmentWord",
-                            [
-                              [
-                                "Name",
-                                "show"
-                              ],
-                              [
-                                "Word",
-                                "echo",
-                                [
-                                  [
-                                    "WordLiteral",
-                                    "echo"
-                                  ]
-                                ]
-                              ]
-                            ]
+                            "WordLiteral",
+                            "show=echo"
                           ]
                         ]
                       ]

tests/golden/good/2.3-token-recognition/2.3.1-alias-substitution/alias-cycle.t/expected.json

@@ -41,29 +41,18 @@
                         "ls='ls -h -t -l'",
                         [
                           [
-                            "WordAssignmentWord",
+                            "WordLiteral",
+                            "ls="
+                          ],
+                          [
+                            "WordSingleQuoted",
                             [
+                              "Word",
+                              "ls -h -t -l",
                               [
-                                "Name",
-                                "ls"
-                              ],
-                              [
-                                "Word",
-                                "'ls -h -t -l'",
                                 [
-                                  [
-                                    "WordSingleQuoted",
-                                    [
-                                      "Word",
-                                      "ls -h -t -l",
-                                      [
-                                        [
-                                          "WordLiteral",
-                                          "ls -h -t -l"
-                                        ]
-                                      ]
-                                    ]
-                                  ]
+                                  "WordLiteral",
+                                  "ls -h -t -l"
                                 ]
                               ]
                             ]