c# - Android speech recognition pass data back to Xamarin Forms

Question

im really stuck right now and im quite new to Xamarin. I use Xamarin Forms to develop an App with speech recognition.

I created just a simple UI with a button and an entry-box.

Working:

• Press button and show popup with speech recognition
• read the spoken words into a var

Not Working:

• pass the data back to the Xamarin Forms UI (entry)

StartPage.xaml.cs:

    private void BtnRecord_OnClicked(object sender, EventArgs e)
    {
        WaitForSpeechToText();
    } 

    private async void WaitForSpeechToText()
    {
        EntrySpeech.Text = await DependencyService.Get<Listener.ISpeechToText>().SpeechToTextAsync();
    }

ISpeechToText.cs:

public interface ISpeechToText
{
    Task<string> SpeechToTextAsync();
}

to call native code.

SpeechToText_Android.cs:

    public class SpeechToText_Android : ISpeechToText
    {
    private const int VOICE = 10;

    public SpeechToText_Android() { }

    public Task<string> SpeechToTextAsync()
    {
        var tcs = new TaskCompletionSource<string>();

        try
        {
            var voiceIntent = new Intent(RecognizerIntent.ActionRecognizeSpeech);
            voiceIntent.PutExtra(RecognizerIntent.ExtraLanguageModel, RecognizerIntent.LanguageModelFreeForm);
            voiceIntent.PutExtra(RecognizerIntent.ExtraPrompt, "Sprechen Sie jetzt");
            voiceIntent.PutExtra(RecognizerIntent.ExtraSpeechInputCompleteSilenceLengthMillis, 1500);
            voiceIntent.PutExtra(RecognizerIntent.ExtraSpeechInputPossiblyCompleteSilenceLengthMillis, 1500);
            voiceIntent.PutExtra(RecognizerIntent.ExtraSpeechInputMinimumLengthMillis, 15000);
            voiceIntent.PutExtra(RecognizerIntent.ExtraMaxResults, 1);
            voiceIntent.PutExtra(RecognizerIntent.ExtraLanguage, Java.Util.Locale.Default);

            try
            {
                ((Activity)Forms.Context).StartActivityForResult(voiceIntent, VOICE);

            }
            catch (ActivityNotFoundException a)
            {
                tcs.SetResult("Device doesn't support speech to text");
            }
        }
        catch (Exception ex)
        {

            tcs.SetException(ex);
        }

        return tcs.Task;
    }
}

MainActivity.cs:

    protected override void OnActivityResult(int requestCode, Result resultVal, Intent data)
    {  
        if (requestCode == VOICE)
        {
            if (resultVal == Result.Ok)
            {
                var matches = data.GetStringArrayListExtra(RecognizerIntent.ExtraResults);
                if (matches.Count != 0)
                {
                    string textInput = matches[0].ToString();
                    if (textInput.Length > 500)
                        textInput = textInput.Substring(0, 500);
                }
                // RETURN 
            }
        }
        base.OnActivityResult(requestCode, resultVal, data);
    }

First i thought that i can pass the result with

return tcs.Task;

back to the ui, but then i noticed that this return is happening when the pop up of the speech recognition has finished to render. At this moment there was not a single word spoken.

The spoken words are in the string "textInput" in the OnActivityResult function, but how can i pass this string back to the Xamarin.Forms UI?

Thanks guys !

Answer 1

I would use an AutoResetEvent to pause the return until the OnActivityResult is called, which would be until the user either records something, cancels, or you timeout their action in the AutoResetEvent.

Return a Task<string> from your SpeechToTextAsync method:

public interface ISpeechToText
{
    Task<string> SpeechToTextAsync();
}

Add an AutoResetEvent to pause execution:

Note: Wrapping the AutoResetEvent.WaitOne to prevent hanging the application looper

public class SpeechToText_Android : Listener.ISpeechToText
{
    public static AutoResetEvent autoEvent = new AutoResetEvent(false);
    public static string SpeechText;
    const int VOICE = 10;

    public async Task<string> SpeechToTextAsync()
    {
        var voiceIntent = new Intent(RecognizerIntent.ActionRecognizeSpeech);
        voiceIntent.PutExtra(RecognizerIntent.ExtraLanguageModel, RecognizerIntent.LanguageModelFreeForm);
        voiceIntent.PutExtra(RecognizerIntent.ExtraPrompt, "Sprechen Sie jetzt");
        voiceIntent.PutExtra(RecognizerIntent.ExtraSpeechInputCompleteSilenceLengthMillis, 1500);
        voiceIntent.PutExtra(RecognizerIntent.ExtraSpeechInputPossiblyCompleteSilenceLengthMillis, 1500);
        voiceIntent.PutExtra(RecognizerIntent.ExtraSpeechInputMinimumLengthMillis, 15000);
        voiceIntent.PutExtra(RecognizerIntent.ExtraMaxResults, 1);
        voiceIntent.PutExtra(RecognizerIntent.ExtraLanguage, Java.Util.Locale.Default);

        SpeechText = "";
        autoEvent.Reset();
        ((Activity)Forms.Context).StartActivityForResult(voiceIntent, VOICE);
        await Task.Run(() => { autoEvent.WaitOne(new TimeSpan(0, 2, 0)); });
        return SpeechText;
    }
}

MainActivity OnActivityResult:

    const int VOICE = 10;
    protected override void OnActivityResult(int requestCode, Result resultCode, Intent data)
    {
        base.OnActivityResult(requestCode, resultCode, data);
        if (requestCode == VOICE)
        {
            if (resultCode == Result.Ok)
            {
                var matches = data.GetStringArrayListExtra(RecognizerIntent.ExtraResults);
                if (matches.Count != 0)
                {
                    var textInput = matches[0];
                    if (textInput.Length > 500)
                        textInput = textInput.Substring(0, 500);
                    SpeechToText_Android.SpeechText = textInput;
                }
            }
            SpeechToText_Android.autoEvent.Set();
        }
    }

Note: This is making use of a couple of static vars to simplify the implementation of this example... Some developers would say this is a code smell, I semi-agree, but you can not ever have more then one Google Speech Recognizer running at a time....

Hello World Example:

public class App : Application
{
    public App()
    {
        var speechTextLabel = new Label
        {
            HorizontalTextAlignment = TextAlignment.Center,
            Text = "Waiting for text"
        };

        var speechButton = new Button();
        speechButton.Text = "Fetch Speech To Text Results";
        speechButton.Clicked += async (object sender, EventArgs e) =>
        {
            var speechText = await WaitForSpeechToText();
            speechTextLabel.Text = string.IsNullOrEmpty(speechText) ? "Nothing Recorded" : speechText;
        };

        var content = new ContentPage
        {
            Title = "Speech",
            Content = new StackLayout
            {
                VerticalOptions = LayoutOptions.Center,
                Children = {
                    new Label {
                        HorizontalTextAlignment = TextAlignment.Center,
                        Text = "Welcome to Xamarin Forms!"
                    },
                    speechButton,
                    speechTextLabel
                }
            }
        };
        MainPage = new NavigationPage(content);
    }

    async Task<string> WaitForSpeechToText()
    {
        return await DependencyService.Get<Listener.ISpeechToText>().SpeechToTextAsync();
    }
}