add tokenization

Author: sfluegel05

chebai/preprocessing/reader.py

@@ -387,15 +387,16 @@ def _read_data(self, raw_data: str) -> Optional[List[int]]:
         try:
             mol = Chem.MolFromSmiles(raw_data, sanitize=False)
             inchi = Chem.MolToInchi(mol)
+            tokenized = [self._get_token_index(v) for v in inchi]
         except Exception:
             print(f"could not process {raw_data}")
             # print(f'\t{e}')
             self.error_count += 1
             print(f"\terror count: {self.error_count}")
-            inchi = None
+            tokenized = None
             # if self.error_count > 20:
             #    raise Exception('Too many errors')
-        return inchi  # one letter = one token
+        return tokenized  # one letter = one token
 
 
 class OrdReader(DataReader):