Skip to content

add methods to get access to the same parameters than BpeOptions #7669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Philippe-Laval wants to merge 1 commit into
base: main
Choose a base branch
from
+85 −0
Open

add methods to get access to the same parameters than BpeOptions #7669

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 85 additions & 0 deletions src/Microsoft.ML.Tokenizers/Model/BPETokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,91 @@ public static async Task<BpeTokenizer> CreateAsync(
return new BpeTokenizer(result.vocab, result.merges, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, endOfWordSuffix, fuseUnknownTokens);
}


/// <summary>
/// Create a new Bpe tokenizer object to use for text encoding.
/// </summary>
/// <param name="vocabStream">The JSON stream containing the dictionary of string keys and their ids.</param>
/// <param name="mergesStream">The stream containing the tokens's pairs list.</param>
/// <param name="preTokenizer">The pre-tokenizer to use.</param>
/// <param name="normalizer">The normalizer to use.</param>
/// <param name="specialTokens">The dictionary mapping special tokens to Ids.</param>
/// <param name="unknownToken"> The unknown token to be used by the model.</param>
/// <param name="continuingSubwordPrefix">The prefix to attach to sub-word units that don’t represent a beginning of word.</param>
/// <param name="endOfWordSuffix">The suffix to attach to sub-word units that represent an end of word.</param>
/// <param name="fuseUnknownTokens">Indicate whether allowing multiple unknown tokens get fused.</param>
/// <param name="byteLevel">Indicate whether to handle the input text in byte level.</param>
/// <param name="beginningOfSentenceToken">The beginning of sentence token.</param>
/// <param name="endOfSentenceToken">The end of sentence token.</param>
/// <remarks>
/// When creating the tokenizer, ensure that the vocabulary stream is sourced from a trusted provider.
/// </remarks>
public static BpeTokenizer Create(
Stream vocabStream,
Stream? mergesStream,
PreTokenizer? preTokenizer = null,
Normalizer? normalizer = null,
IReadOnlyDictionary<string, int>? specialTokens = null,
string? unknownToken = null,
string? continuingSubwordPrefix = null,
string? endOfWordSuffix = null,
bool fuseUnknownTokens = false,
bool byteLevel = false,
string? beginningOfSentenceToken = null,
string? endOfSentenceToken = null)
{
if (vocabStream is null)
{
throw new ArgumentNullException(nameof(vocabStream));
}

(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();

return new BpeTokenizer(result.vocab, result.merges, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, endOfWordSuffix, fuseUnknownTokens, byteLevel, beginningOfSentenceToken, endOfSentenceToken);
}

/// <summary>
/// Create a new Bpe tokenizer object asynchronously to use for text encoding.
/// </summary>
/// <param name="vocabStream">The JSON stream containing the dictionary of string keys and their ids.</param>
/// <param name="mergesStream">The stream containing the tokens's pairs list.</param>
/// <param name="preTokenizer">The pre-tokenizer to use.</param>
/// <param name="normalizer">The normalizer to use.</param>
/// <param name="specialTokens">The dictionary mapping special tokens to Ids.</param>
/// <param name="unknownToken"> The unknown token to be used by the model.</param>
/// <param name="continuingSubwordPrefix">The prefix to attach to sub-word units that don’t represent a beginning of word.</param>
/// <param name="endOfWordSuffix">The suffix to attach to sub-word units that represent an end of word.</param>
/// <param name="fuseUnknownTokens">Indicate whether allowing multiple unknown tokens get fused.</param>
/// <param name="byteLevel">Indicate whether to handle the input text in byte level.</param>
/// <param name="beginningOfSentenceToken">The beginning of sentence token.</param>
/// <param name="endOfSentenceToken">The end of sentence token.</param>
/// <remarks>
/// When creating the tokenizer, ensure that the vocabulary stream is sourced from a trusted provider.
/// </remarks>
public static async Task<BpeTokenizer> CreateAsync(
Stream vocabStream,
Stream? mergesStream,
PreTokenizer? preTokenizer = null,
Normalizer? normalizer = null,
IReadOnlyDictionary<string, int>? specialTokens = null,
string? unknownToken = null,
string? continuingSubwordPrefix = null,
string? endOfWordSuffix = null,
bool fuseUnknownTokens = false,
bool byteLevel = false,
string? beginningOfSentenceToken = null,
string? endOfSentenceToken = null)
{
if (vocabStream is null)
{
throw new ArgumentNullException(nameof(vocabStream));
}

(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false);

return new BpeTokenizer(result.vocab, result.merges, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, endOfWordSuffix, fuseUnknownTokens, byteLevel, beginningOfSentenceToken, endOfSentenceToken);
}

/// <summary>
/// Construct a new Bpe model object to use for text encoding.
/// </summary>
Expand Down
Loading