Embedding models fail possibly on scientific characters no matter how the pdf is parsed
I'm trying to create a RAG pipeline using Llamaindex. Anytime I try to use an embedding model on these papers for semantic chunking or anything, it doesn't follow through, which is possibly due to the presence of scientific characters. Are there any options other than removing the chunks/papers that are problematic? The documents have been parsed in Llamaparse and saved in markdown all with SimpleDirectoryReader - same issue arose in the past when I parsed with PyMuPDF and SimpleDirectoryReader, so my thoughts go towards the embedding process as the main issue. The parsed data itself has been properly preserved in the .pkl file so I don't think any issues with reading that. It works on simple texts, or docs[0] when tested individually but the entire process fails together.
from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
import pickle
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
import nest_asyncio
nest_asyncio.apply()
from dotenv import load_dotenv
import os
load_dotenv()
embed_model = HuggingFaceEmbedding(
model_name="BAAI/bge-small-en-v1.5"
)
with open("/Users/rifah/Desktop/p/species/mds.pkl", "rb") as f:
docs = pickle.load(f)
splitter = SemanticSplitterNodeParser(buffer_size=1, embed_model=embed_model,include_metadata=True)
nodes = splitter.get_nodes_from_documents(docs)
2026-07-24 16:27:07.093685: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/opt/anaconda3/lib/python3.12/site-packages/keras/src/export/tf2onnx_lib.py:8: FutureWarning: In the future `np.object` will be defined as the corresponding NumPy scalar.
if not hasattr(np, "object"):
Embedding attempt failed: TextEncodeInput must be Union[TextInputSequence, Tuple[InputSequence, InputSequence]]
Embedding attempt failed: TextEncodeInput must be Union[TextInputSequence, Tuple[InputSequence, InputSequence]]
Embedding attempt failed: TextEncodeInput must be Union[TextInputSequence, Tuple[InputSequence, InputSequence]]
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/model.py:557, in BaseModel.preprocess(self, inputs, prompt, **kwargs)
556 try:
--> 557 preprocessed = self[0].preprocess(inputs, prompt=prompt, **kwargs)
558 except TypeError:
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/modules/transformer.py:988, in Transformer.preprocess(self, inputs, prompt, processing_kwargs, **kwargs)
987 with suggest_extra_on_exception():
--> 988 processor_output = self._call_processor(
989 modality,
990 processor_inputs,
991 modality_kwargs,
992 common_kwargs,
993 chat_template_kwargs=chat_template_kwargs,
994 )
996 if num_images_per_sample is not None and "image_grid_thw" in processor_output:
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/modules/transformer.py:1241, in Transformer._call_processor(self, modality, processor_inputs, modality_kwargs, common_kwargs, chat_template_kwargs)
1239 return self._call_multimodal_processor(modality, processor_inputs, modality_kwargs, common_kwargs)
-> 1241 return self._call_single_modality_processor(modality, processor_inputs, modality_kwargs, common_kwargs)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/modules/transformer.py:1302, in Transformer._call_single_modality_processor(self, modality, processor_inputs, modality_kwargs, common_kwargs)
1301 primary_input = processor_inputs.pop(modality_type)
-> 1302 return self.processor(primary_input, **processor_inputs, **call_kwargs)
1303 return self.processor(**processor_inputs, **call_kwargs)
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:3078, in PreTrainedTokenizerBase.__call__(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)
3077 self._switch_to_input_mode()
-> 3078 encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)
3079 if text_target is not None:
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:3166, in PreTrainedTokenizerBase._call_one(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
3165 batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
-> 3166 return self.batch_encode_plus(
3167 batch_text_or_text_pairs=batch_text_or_text_pairs,
3168 add_special_tokens=add_special_tokens,
3169 padding=padding,
3170 truncation=truncation,
3171 max_length=max_length,
3172 stride=stride,
3173 is_split_into_words=is_split_into_words,
3174 pad_to_multiple_of=pad_to_multiple_of,
3175 padding_side=padding_side,
3176 return_tensors=return_tensors,
3177 return_token_type_ids=return_token_type_ids,
3178 return_attention_mask=return_attention_mask,
3179 return_overflowing_tokens=return_overflowing_tokens,
3180 return_special_tokens_mask=return_special_tokens_mask,
3181 return_offsets_mapping=return_offsets_mapping,
3182 return_length=return_length,
3183 verbose=verbose,
3184 split_special_tokens=split_special_tokens,
3185 **kwargs,
3186 )
3187 else:
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:3367, in PreTrainedTokenizerBase.batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
3358 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
3359 padding=padding,
3360 truncation=truncation,
(...)
3364 **kwargs,
3365 )
-> 3367 return self._batch_encode_plus(
3368 batch_text_or_text_pairs=batch_text_or_text_pairs,
3369 add_special_tokens=add_special_tokens,
3370 padding_strategy=padding_strategy,
3371 truncation_strategy=truncation_strategy,
3372 max_length=max_length,
3373 stride=stride,
3374 is_split_into_words=is_split_into_words,
3375 pad_to_multiple_of=pad_to_multiple_of,
3376 padding_side=padding_side,
3377 return_tensors=return_tensors,
3378 return_token_type_ids=return_token_type_ids,
3379 return_attention_mask=return_attention_mask,
3380 return_overflowing_tokens=return_overflowing_tokens,
3381 return_special_tokens_mask=return_special_tokens_mask,
3382 return_offsets_mapping=return_offsets_mapping,
3383 return_length=return_length,
3384 verbose=verbose,
3385 split_special_tokens=split_special_tokens,
3386 **kwargs,
3387 )
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_fast.py:553, in PreTrainedTokenizerFast._batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens)
551 self._tokenizer.encode_special_tokens = split_special_tokens
--> 553 encodings = self._tokenizer.encode_batch(
554 batch_text_or_text_pairs,
555 add_special_tokens=add_special_tokens,
556 is_pretokenized=is_split_into_words,
557 )
559 # Convert encoding to dict
560 # `Tokens` has type: tuple[
561 # list[dict[str, list[list[int]]]] or list[dict[str, 2D-Tensor]],
562 # list[EncodingFast]
563 # ]
564 # with nested dimensions corresponding to batch, overflows, sequence length
TypeError: TextEncodeInput must be Union[TextInputSequence, Tuple[InputSequence, InputSequence]]
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
Cell In[4], line 21
15 docs = pickle.load(f)
18 splitter = SemanticSplitterNodeParser(buffer_size=1, embed_model=embed_model,include_metadata=True)
---> 21 nodes = splitter.get_nodes_from_documents(docs)
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/core/node_parser/interface.py:176, in NodeParser.get_nodes_from_documents(self, documents, show_progress, **kwargs)
171 doc_id_to_document = {doc.id_: doc for doc in documents}
173 with self.callback_manager.event(
174 CBEventType.NODE_PARSING, payload={EventPayload.DOCUMENTS: documents}
175 ) as event:
--> 176 nodes = self._parse_nodes(documents, show_progress=show_progress, **kwargs)
177 nodes = self._postprocess_parsed_nodes(nodes, doc_id_to_document)
179 event.on_end({EventPayload.NODES: nodes})
File /opt/anaconda3/lib/python3.12/site-packages/llama_index_instrumentation/dispatcher.py:413, in Dispatcher.span..wrapper(func, instance, args, kwargs)
410 _logger.debug(f"Failed to reset active_span_id: {e}")
412 try:
--> 413 result = func(*args, **kwargs)
414 if isinstance(result, asyncio.Future):
415 # If the result is a Future, wrap it
416 new_future = asyncio.ensure_future(result)
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/core/node_parser/text/semantic_splitter.py:137, in SemanticSplitterNodeParser._parse_nodes(self, nodes, show_progress, **kwargs)
134 nodes_with_progress = get_tqdm_iterable(nodes, show_progress, "Parsing nodes")
136 for node in nodes_with_progress:
--> 137 nodes = self.build_semantic_nodes_from_documents([node], show_progress)
138 all_nodes.extend(nodes)
140 return all_nodes
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/core/node_parser/text/semantic_splitter.py:173, in SemanticSplitterNodeParser.build_semantic_nodes_from_documents(self, documents, show_progress)
169 text_splits = self.sentence_splitter(text)
171 sentences = self._build_sentence_groups(text_splits)
--> 173 combined_sentence_embeddings = self.embed_model.get_text_embedding_batch(
174 [s["combined_sentence"] for s in sentences],
175 show_progress=show_progress,
176 )
178 for i, embedding in enumerate(combined_sentence_embeddings):
179 sentences[i]["combined_sentence_embedding"] = embedding
File /opt/anaconda3/lib/python3.12/site-packages/llama_index_instrumentation/dispatcher.py:413, in Dispatcher.span..wrapper(func, instance, args, kwargs)
410 _logger.debug(f"Failed to reset active_span_id: {e}")
412 try:
--> 413 result = func(*args, **kwargs)
414 if isinstance(result, asyncio.Future):
415 # If the result is a Future, wrap it
416 new_future = asyncio.ensure_future(result)
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/core/base/embeddings/base.py:517, in BaseEmbedding.get_text_embedding_batch(self, texts, show_progress, **kwargs)
515 self.rate_limiter.acquire()
516 if not self.embeddings_cache:
--> 517 embeddings = self._get_text_embeddings(cur_batch)
518 elif self.embeddings_cache is not None:
519 embeddings = self._get_text_embeddings_cached(cur_batch)
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/embeddings/huggingface/base.py:333, in HuggingFaceEmbedding._get_text_embeddings(self, texts)
322 def _get_text_embeddings(self, texts: List[str]) -> List[List[float]]:
323 """
324 Generates Embeddings for text.
325
(...)
331
332 """
--> 333 return self._embed(texts, prompt_name="text")
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/embeddings/huggingface/base.py:268, in HuggingFaceEmbedding._embed(self, inputs, prompt_name)
248 def _embed(
249 self,
250 inputs: List[Union[str, BytesIO]],
251 prompt_name: Optional[str] = None,
252 ) -> List[List[float]]:
253 """
254 Generates Embeddings with input validation and retry mechanism.
255
(...)
266
267 """
--> 268 return self._embed_with_retry(inputs, prompt_name)
File /opt/anaconda3/lib/python3.12/site-packages/tenacity/__init__.py:331, in BaseRetrying.wraps..wrapped_f(*args, **kw)
329 copy = self.copy()
330 wrapped_f.statistics = copy.statistics # type: ignore[attr-defined]
--> 331 return copy(f, *args, **kw)
File /opt/anaconda3/lib/python3.12/site-packages/tenacity/__init__.py:470, in Retrying.__call__(self, fn, *args, **kwargs)
468 retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs)
469 while True:
--> 470 do = self.iter(retry_state=retry_state)
471 if isinstance(do, DoAttempt):
472 try:
File /opt/anaconda3/lib/python3.12/site-packages/tenacity/__init__.py:371, in BaseRetrying.iter(self, retry_state)
369 result = None
370 for action in self.iter_state.actions:
--> 371 result = action(retry_state)
372 return result
File /opt/anaconda3/lib/python3.12/site-packages/tenacity/__init__.py:413, in BaseRetrying._post_stop_check_actions..exc_check(rs)
411 retry_exc = self.retry_error_cls(fut)
412 if self.reraise:
--> 413 raise retry_exc.reraise()
414 raise retry_exc from fut.exception()
File /opt/anaconda3/lib/python3.12/site-packages/tenacity/__init__.py:184, in RetryError.reraise(self)
182 def reraise(self) -> t.NoReturn:
183 if self.last_attempt.failed:
--> 184 raise self.last_attempt.result()
185 raise self
File /opt/anaconda3/lib/python3.12/concurrent/futures/_base.py:449, in Future.result(self, timeout)
447 raise CancelledError()
448 elif self._state == FINISHED:
--> 449 return self.__get_result()
451 self._condition.wait(timeout)
453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
File /opt/anaconda3/lib/python3.12/concurrent/futures/_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> 401 raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None
File /opt/anaconda3/lib/python3.12/site-packages/tenacity/__init__.py:473, in Retrying.__call__(self, fn, *args, **kwargs)
471 if isinstance(do, DoAttempt):
472 try:
--> 473 result = fn(*args, **kwargs)
474 except BaseException: # noqa: B902
475 retry_state.set_exception(sys.exc_info()) # type: ignore[arg-type]
File /opt/anaconda3/lib/python3.12/site-packages/llama_index/embeddings/huggingface/base.py:236, in HuggingFaceEmbedding._embed_with_retry(self, inputs, prompt_name)
234 self._model.stop_multi_process_pool(pool=pool)
235 else:
--> 236 emb = self._model.encode(
237 inputs,
238 batch_size=self.embed_batch_size,
239 prompt_name=prompt_name,
240 normalize_embeddings=self.normalize,
241 show_progress_bar=self.show_progress_bar,
242 )
243 return emb.tolist()
244 except Exception as e:
File /opt/anaconda3/lib/python3.12/site-packages/torch/utils/_contextlib.py:115, in context_decorator..decorate_context(*args, **kwargs)
112 @functools.wraps(func)
113 def decorate_context(*args, **kwargs):
114 with ctx_factory():
--> 115 return func(*args, **kwargs)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/util/decorators.py:41, in deprecated_kwargs..decorator..wrapper(*args, **kwargs)
39 else:
40 kwargs.pop(old_name)
---> 41 return func(*args, **kwargs)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/sentence_transformer/model.py:654, in SentenceTransformer.encode(self, inputs, prompt_name, prompt, batch_size, show_progress_bar, output_value, precision, convert_to_numpy, convert_to_tensor, device, normalize_embeddings, truncate_dim, pool, chunk_size, **kwargs)
652 for start_index in trange(0, len(inputs_sorted), batch_size, desc="Batches", disable=not show_progress_bar):
653 inputs_batch = inputs_sorted[start_index : start_index + batch_size]
--> 654 features = self.preprocess(inputs_batch, prompt=prompt, **kwargs)
656 if is_hpu:
657 features = self._pad_features_for_hpu(features)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/model.py:561, in BaseModel.preprocess(self, inputs, prompt, **kwargs)
559 if prompt and modality == "text":
560 inputs = [(prompt + inp[0],) + inp[1:] if isinstance(inp, tuple) else prompt + inp for inp in inputs]
--> 561 preprocessed = self[0].preprocess(inputs, **kwargs)
562 except AttributeError:
563 if prompt and modality == "text":
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/modules/transformer.py:988, in Transformer.preprocess(self, inputs, prompt, processing_kwargs, **kwargs)
985 num_images_per_sample, num_videos_per_sample = _count_media_per_sample(processor_inputs["message"])
987 with suggest_extra_on_exception():
--> 988 processor_output = self._call_processor(
989 modality,
990 processor_inputs,
991 modality_kwargs,
992 common_kwargs,
993 chat_template_kwargs=chat_template_kwargs,
994 )
996 if num_images_per_sample is not None and "image_grid_thw" in processor_output:
997 processor_output["num_images_per_sample"] = torch.tensor(num_images_per_sample, dtype=torch.long)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/modules/transformer.py:1241, in Transformer._call_processor(self, modality, processor_inputs, modality_kwargs, common_kwargs, chat_template_kwargs)
1238 if isinstance(self.processor, ProcessorMixin):
1239 return self._call_multimodal_processor(modality, processor_inputs, modality_kwargs, common_kwargs)
-> 1241 return self._call_single_modality_processor(modality, processor_inputs, modality_kwargs, common_kwargs)
File /opt/anaconda3/lib/python3.12/site-packages/sentence_transformers/base/modules/transformer.py:1302, in Transformer._call_single_modality_processor(self, modality, processor_inputs, modality_kwargs, common_kwargs)
1300 if modality_type in processor_inputs:
1301 primary_input = processor_inputs.pop(modality_type)
-> 1302 return self.processor(primary_input, **processor_inputs, **call_kwargs)
1303 return self.processor(**processor_inputs, **call_kwargs)
1305 raise RuntimeError(
1306 f"Could not determine how to call processor of type {type(self.processor).__name__} "
1307 f"for modality '{format_modality(modality)}'"
1308 )
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:3078, in PreTrainedTokenizerBase.__call__(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)
3076 if not self._in_target_context_manager:
3077 self._switch_to_input_mode()
-> 3078 encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)
3079 if text_target is not None:
3080 self._switch_to_target_mode()
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:3166, in PreTrainedTokenizerBase._call_one(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
3161 raise ValueError(
3162 f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
3163 f" {len(text_pair)}."
3164 )
3165 batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
-> 3166 return self.batch_encode_plus(
3167 batch_text_or_text_pairs=batch_text_or_text_pairs,
3168 add_special_tokens=add_special_tokens,
3169 padding=padding,
3170 truncation=truncation,
3171 max_length=max_length,
3172 stride=stride,
3173 is_split_into_words=is_split_into_words,
3174 pad_to_multiple_of=pad_to_multiple_of,
3175 padding_side=padding_side,
3176 return_tensors=return_tensors,
3177 return_token_type_ids=return_token_type_ids,
3178 return_attention_mask=return_attention_mask,
3179 return_overflowing_tokens=return_overflowing_tokens,
3180 return_special_tokens_mask=return_special_tokens_mask,
3181 return_offsets_mapping=return_offsets_mapping,
3182 return_length=return_length,
3183 verbose=verbose,
3184 split_special_tokens=split_special_tokens,
3185 **kwargs,
3186 )
3187 else:
3188 return self.encode_plus(
3189 text=text,
3190 text_pair=text_pair,
(...)
3208 **kwargs,
3209 )
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:3367, in PreTrainedTokenizerBase.batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
3357 # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
3358 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
3359 padding=padding,
3360 truncation=truncation,
(...)
3364 **kwargs,
3365 )
-> 3367 return self._batch_encode_plus(
3368 batch_text_or_text_pairs=batch_text_or_text_pairs,
3369 add_special_tokens=add_special_tokens,
3370 padding_strategy=padding_strategy,
3371 truncation_strategy=truncation_strategy,
3372 max_length=max_length,
3373 stride=stride,
3374 is_split_into_words=is_split_into_words,
3375 pad_to_multiple_of=pad_to_multiple_of,
3376 padding_side=padding_side,
3377 return_tensors=return_tensors,
3378 return_token_type_ids=return_token_type_ids,
3379 return_attention_mask=return_attention_mask,
3380 return_overflowing_tokens=return_overflowing_tokens,
3381 return_special_tokens_mask=return_special_tokens_mask,
3382 return_offsets_mapping=return_offsets_mapping,
3383 return_length=return_length,
3384 verbose=verbose,
3385 split_special_tokens=split_special_tokens,
3386 **kwargs,
3387 )
File /opt/anaconda3/lib/python3.12/site-packages/transformers/tokenization_utils_fast.py:553, in PreTrainedTokenizerFast._batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens)
550 if self._tokenizer.encode_special_tokens != split_special_tokens:
551 self._tokenizer.encode_special_tokens = split_special_tokens
--> 553 encodings = self._tokenizer.encode_batch(
554 batch_text_or_text_pairs,
555 add_special_tokens=add_special_tokens,
556 is_pretokenized=is_split_into_words,
557 )
559 # Convert encoding to dict
560 # `Tokens` has type: tuple[
561 # list[dict[str, list[list[int]]]] or list[dict[str, 2D-Tensor]],
562 # list[EncodingFast]
563 # ]
564 # with nested dimensions corresponding to batch, overflows, sequence length
565 tokens_and_encodings = [
566 self._convert_encoding(
567 encoding=encoding,
(...)
576 for encoding in encodings
577 ]
TypeError: TextEncodeInput must be Union[TextInputSequence, Tuple[InputSequence, InputSequence]]
Any fixes or workarounds?