Skip to content

Commit

Permalink
Fix percent tokenization
Browse files Browse the repository at this point in the history
  • Loading branch information
SubhadeepJasu committed Jan 31, 2025
1 parent 19b1ec7 commit 32e7f24
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 3 deletions.
1 change: 1 addition & 0 deletions src/core/scientific_calculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ def __init__(self, data: str, memory: ContextualMemory, tokenize: bool=True):
self.angle_mode = self.input_dict['angleMode']
if tokenize:
self.tokens = Tokenizer.st_tokenize(self.input_dict['input'])
print ('Tokens: ', self.tokens)

self.substitute_value: float | complex = 0
self.zero_limit = False
Expand Down
9 changes: 6 additions & 3 deletions src/core/tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def _space_removal(original: str) -> str:

@staticmethod
def _is_number(exp: str) -> bool:
return exp.endswith(tuple([str(i) for i in range(1, 10)] + ['.', 'x']))
return exp.endswith(tuple([str(i) for i in range(0, 10)] + ['.', 'x']))


@staticmethod
Expand Down Expand Up @@ -84,7 +84,7 @@ def _unary_minus_convert(exp: str):
if i < len(tokens):
tokens [i] = '( 0 u'
tokens [i + 1] = tokens [i + 1] + " )"
elif tokens [i - 1] == ')' or tokens [i - 1] == 'x' or Tokenizer._is_number (tokens [i - 1]):
elif tokens [i - 1] == ')' or tokens [i - 1] == 'x' or Tokenizer._is_number (tokens [i - 1].strip()):
tokens [i] = '-'
else:
if i < len(tokens):
Expand Down Expand Up @@ -275,8 +275,11 @@ def st_tokenize(input:str) -> list[str]:

# Intelligently convert expressions based on common rules
exp = Tokenizer._algebraic_parenthesis_product_convert(exp)
exp = Tokenizer._unary_minus_convert(exp)
print(exp)
exp = Tokenizer._relative_percentage_convert(exp)
print(exp)
exp = Tokenizer._unary_minus_convert(exp)
print(exp)
exp = Tokenizer._space_removal(exp.strip())

return exp.split(' ')
Expand Down

0 comments on commit 32e7f24

Please sign in to comment.