Add a test

uiuc-focal-lab · Aug 4, 2024 · ebdfbe4 · ebdfbe4
1 parent d681190
commit ebdfbe4
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 2 deletions.
diff --git a/syncode/parse_result.py b/syncode/parse_result.py
@@ -61,15 +61,17 @@ def from_accept_terminals(cur_accept_terminals, next_accept_terminals, remainder
                 if t == final_terminal:
                     for t2 in next_accept_terminals:
                         accept_sequences.add(AcceptSequence([final_terminal, t2]))
+
                     if ignore_terminals is not None:
+                        # Since ignore terminals are allowed anywhere in the code (final terminal, ignore terminal) is also a valid accept sequence
                         for tignore in ignore_terminals:
                             accept_sequences.add(AcceptSequence([final_terminal, tignore]))
 
                         # These 3 length accept sequences are useful in under-approximating 
                         # `grammar_strict` mode as they help improve the precision of SynCode
                         for tignore in ignore_terminals:
                             for t2 in next_accept_terminals:
-                                accept_sequences.add(AcceptSequence([final_terminal, tignore, t]))
+                                accept_sequences.add(AcceptSequence([final_terminal, tignore, t2]))
                 else:
                     accept_sequences.add(AcceptSequence([t]))
 

diff --git a/tests/test_misc.py b/tests/test_misc.py
@@ -59,7 +59,16 @@ def test_mask_store_misc2(self):
         dfa_mask = DFAMaskStore.load_dfa_mask_store(grammar=grammar, tokenizer=tokenizer, use_cache=False, logger=common.EmptyLogger())
         mask = dfa_mask.get_accept_mask(r, get_list=True)
         self.assertIn(' have', mask)
-
+
+    def test_mask_store_misc3(self):
+        grammar = Grammar(TestParserMisc.essay_grammar())
+        model = 'microsoft/Phi-3-mini-128k-instruct'
+        tokenizer = common.load_tokenizer(model)
+        inc_parser = create_parser(grammar)
+        r = inc_parser.get_acceptable_next_terminals("I have been working there for 5 years.")
+        dfa_mask = DFAMaskStore.load_dfa_mask_store(grammar=grammar, tokenizer=tokenizer, use_cache=False, logger=common.EmptyLogger())
+        mask = dfa_mask.get_accept_mask(r, get_list=True)
+        self.assertIn(' I', mask)
 
     def test_parser_calc(self):
         inc_parser = create_parser(Grammar('calc'))