fix(api): do not parse tokens when removing them
This commit is contained in:
parent
14ddfb43bb
commit
1bdfb4ac8a
|
@ -3,7 +3,7 @@ from copy import deepcopy
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
from math import ceil
|
from math import ceil
|
||||||
from re import Pattern, compile
|
from re import Pattern, compile
|
||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
|
@ -515,9 +515,13 @@ def parse_reseed(prompt: str) -> Tuple[str, List[Reseed]]:
|
||||||
return get_tokens_from_prompt(prompt, RESEED_TOKEN, parser=parse_reseed_group)
|
return get_tokens_from_prompt(prompt, RESEED_TOKEN, parser=parse_reseed_group)
|
||||||
|
|
||||||
|
|
||||||
|
def skip_group(group) -> Any:
|
||||||
|
return group
|
||||||
|
|
||||||
|
|
||||||
def remove_tokens(prompt: Optional[str]) -> Optional[str]:
|
def remove_tokens(prompt: Optional[str]) -> Optional[str]:
|
||||||
if prompt is None:
|
if prompt is None:
|
||||||
return prompt
|
return prompt
|
||||||
|
|
||||||
remainder, tokens = get_tokens_from_prompt(prompt, ANY_TOKEN)
|
remainder, tokens = get_tokens_from_prompt(prompt, ANY_TOKEN, parser=skip_group)
|
||||||
return remainder
|
return remainder
|
||||||
|
|
Loading…
Reference in New Issue