From ebdfbe4bfe4584e9433f9b6ef515490574964cca Mon Sep 17 00:00:00 2001 From: Shubham Ugare Date: Sun, 4 Aug 2024 01:33:05 -0500 Subject: [PATCH] Add a test --- syncode/parse_result.py | 4 +++- tests/test_misc.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/syncode/parse_result.py b/syncode/parse_result.py index 830aca5e..6f6a6cc5 100644 --- a/syncode/parse_result.py +++ b/syncode/parse_result.py @@ -61,7 +61,9 @@ def from_accept_terminals(cur_accept_terminals, next_accept_terminals, remainder if t == final_terminal: for t2 in next_accept_terminals: accept_sequences.add(AcceptSequence([final_terminal, t2])) + if ignore_terminals is not None: + # Since ignore terminals are allowed anywhere in the code (final terminal, ignore terminal) is also a valid accept sequence for tignore in ignore_terminals: accept_sequences.add(AcceptSequence([final_terminal, tignore])) @@ -69,7 +71,7 @@ def from_accept_terminals(cur_accept_terminals, next_accept_terminals, remainder # `grammar_strict` mode as they help improve the precision of SynCode for tignore in ignore_terminals: for t2 in next_accept_terminals: - accept_sequences.add(AcceptSequence([final_terminal, tignore, t])) + accept_sequences.add(AcceptSequence([final_terminal, tignore, t2])) else: accept_sequences.add(AcceptSequence([t])) diff --git a/tests/test_misc.py b/tests/test_misc.py index ab74a2ae..1ded4383 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -59,7 +59,16 @@ def test_mask_store_misc2(self): dfa_mask = DFAMaskStore.load_dfa_mask_store(grammar=grammar, tokenizer=tokenizer, use_cache=False, logger=common.EmptyLogger()) mask = dfa_mask.get_accept_mask(r, get_list=True) self.assertIn(' have', mask) - + + def test_mask_store_misc3(self): + grammar = Grammar(TestParserMisc.essay_grammar()) + model = 'microsoft/Phi-3-mini-128k-instruct' + tokenizer = common.load_tokenizer(model) + inc_parser = create_parser(grammar) + r = inc_parser.get_acceptable_next_terminals("I have been working there for 5 years.") + dfa_mask = DFAMaskStore.load_dfa_mask_store(grammar=grammar, tokenizer=tokenizer, use_cache=False, logger=common.EmptyLogger()) + mask = dfa_mask.get_accept_mask(r, get_list=True) + self.assertIn(' I', mask) def test_parser_calc(self): inc_parser = create_parser(Grammar('calc'))