Minor fixes

This commit is contained in:
oobabooga
2023-04-11 11:24:55 -03:00
parent d0a6b53b4b
commit 98dcfb8e12
3 changed files with 15 additions and 12 deletions

View File

@@ -69,6 +69,7 @@ def generate_softprompt_input_tensors(input_ids):
# filler_input_ids += shared.model.config.bos_token_id # setting dummy input_ids to bos tokens
return inputs_embeds, filler_input_ids
# Removes empty replies from gpt4chan outputs
def fix_gpt4chan(s):
for i in range(10):
@@ -77,6 +78,7 @@ def fix_gpt4chan(s):
s = re.sub("--- [0-9]*\n\n\n---", "---", s)
return s
# Fix the LaTeX equations in galactica
def fix_galactica(s):
s = s.replace(r'\[', r'$')