function setTokensProvider(languageId, provider) {
    const languageService = StandaloneServices.get(ILanguageService);
    if (!languageService.isRegisteredLanguageId(languageId)) {
        throw new Error(`Cannot set tokens provider for unknown language ${languageId}`);
    }
    if (isThenable(provider)) {
        return registerTokensProviderFactory(languageId, { create: () => provider });
    }
    return languages.TokenizationRegistry.register(languageId, createTokenizationSupportAdapter(languageId, provider));
}
Set the tokens provider for a language (manual implementation). This tokenizer will be exclusive
with a tokenizer created using `setMonarchTokensProvider`, or with `registerTokensProviderFactory`,
but will work together with a tokens provider set using `registerDocumentSemanticTokensProvider`
or `registerDocumentRangeSemanticTokensProvider`.