diff --git a/src/ui/android/.classpath b/src/ui/android/.classpath index 2f27127f0..7bc01d9a9 100644 --- a/src/ui/android/.classpath +++ b/src/ui/android/.classpath @@ -3,11 +3,7 @@ - - - - - + diff --git a/src/ui/android/.project b/src/ui/android/.project index abf063e43..2773e5ef0 100644 --- a/src/ui/android/.project +++ b/src/ui/android/.project @@ -1,6 +1,6 @@ - GFTranslator + GFVoiceExample diff --git a/src/ui/android/AndroidManifest.xml b/src/ui/android/AndroidManifest.xml index 9fef8e112..9728b71aa 100644 --- a/src/ui/android/AndroidManifest.xml +++ b/src/ui/android/AndroidManifest.xml @@ -2,19 +2,21 @@ + android:versionName="1.0" > + android:targetSdkVersion="18" /> + + + android:theme="@style/AppTheme" > diff --git a/src/ui/android/README b/src/ui/android/README new file mode 100644 index 000000000..1d3bda521 --- /dev/null +++ b/src/ui/android/README @@ -0,0 +1,66 @@ += Overview = + +This directory contains a sample Android app tht uses +the Android speech recognition and TTS APIs along with +JNI bindings to the C PGF runtime to implement a simple +speech translation app. + + += Requirements = + +1. Android SDK: http://developer.android.com/sdk/ + installed in $ANDROID_SDK_LOCATION + +2. Android NDK: http://developer.android.com/tools/sdk/ndk/ + installed in $ANDROID_NDK_LOCATION + += Building = + +Set up Android project: + +# Creates local.properties, not to be checked in +$ $ANDROID_SDK_LOCATION/tools/android update project -p . + +Build libs/libjpgf.jar: + +$ (cd ../../runtime/java && javac org/grammaticalframework/pgf/*.java && jar -cf libjpgf.jar org/grammaticalframework/pgf/*.class) +$ cp ../../runtime/java/libjpgf.jar libs + +Build JNI code: + +$ cd jni +$ $ANDROID_NDK_LOCATION/ndk-build + + +Build APK: + +$ ant debug + + +Install on your device: + +$ ant debug install + +or: + +$ adb install -r bin/MainActivity-debug.apk + + += Changing the grammar = + +1. Replace assets/ResourceDemo.pgf + +2. Edit Translator.java to point to the new file and include its metadata + + += Developing in Eclipse = + +1. Install Android ADT + +2. Eclipse > File > Import > Existing Projects into Workspace > Next + +3. Select root directory... + +4. Select GF/src/ui/android + +5. Finish \ No newline at end of file diff --git a/src/ui/android/assets/ResourceDemo.pgf b/src/ui/android/assets/ResourceDemo.pgf new file mode 100644 index 000000000..c40f38a1d Binary files /dev/null and b/src/ui/android/assets/ResourceDemo.pgf differ diff --git a/src/ui/android/build.xml b/src/ui/android/build.xml new file mode 100644 index 000000000..a10a91491 --- /dev/null +++ b/src/ui/android/build.xml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ui/android/libs/android-support-v4.jar b/src/ui/android/libs/android-support-v4.jar new file mode 100644 index 000000000..cf12d2839 Binary files /dev/null and b/src/ui/android/libs/android-support-v4.jar differ diff --git a/src/ui/android/libs/armeabi/libjpgf.so b/src/ui/android/libs/armeabi/libjpgf.so new file mode 100644 index 000000000..69014824d Binary files /dev/null and b/src/ui/android/libs/armeabi/libjpgf.so differ diff --git a/src/ui/android/libs/libjpgf.jar b/src/ui/android/libs/libjpgf.jar new file mode 100644 index 000000000..e472505d3 Binary files /dev/null and b/src/ui/android/libs/libjpgf.jar differ diff --git a/src/ui/android/proguard-project.txt b/src/ui/android/proguard-project.txt new file mode 100644 index 000000000..f2fe1559a --- /dev/null +++ b/src/ui/android/proguard-project.txt @@ -0,0 +1,20 @@ +# To enable ProGuard in your project, edit project.properties +# to define the proguard.config property as described in that file. +# +# Add project specific ProGuard rules here. +# By default, the flags in this file are appended to flags specified +# in ${sdk.dir}/tools/proguard/proguard-android.txt +# You can edit the include path and order by changing the ProGuard +# include property in project.properties. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# Add any project specific keep options here: + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} diff --git a/src/ui/android/res/drawable-hdpi/ic_action_switch.png b/src/ui/android/res/drawable-hdpi/ic_action_switch.png new file mode 100644 index 000000000..5449a32b8 Binary files /dev/null and b/src/ui/android/res/drawable-hdpi/ic_action_switch.png differ diff --git a/src/ui/android/res/drawable-hdpi/ic_launcher.png b/src/ui/android/res/drawable-hdpi/ic_launcher.png new file mode 100644 index 000000000..a6c350aea Binary files /dev/null and b/src/ui/android/res/drawable-hdpi/ic_launcher.png differ diff --git a/src/ui/android/res/drawable-hdpi/ic_mic.png b/src/ui/android/res/drawable-hdpi/ic_mic.png new file mode 100644 index 000000000..f79ff489b Binary files /dev/null and b/src/ui/android/res/drawable-hdpi/ic_mic.png differ diff --git a/src/ui/android/res/drawable-mdpi/ic_action_switch.png b/src/ui/android/res/drawable-mdpi/ic_action_switch.png new file mode 100644 index 000000000..ecf7d0347 Binary files /dev/null and b/src/ui/android/res/drawable-mdpi/ic_action_switch.png differ diff --git a/src/ui/android/res/drawable-mdpi/ic_launcher.png b/src/ui/android/res/drawable-mdpi/ic_launcher.png new file mode 100644 index 000000000..204c58a8f Binary files /dev/null and b/src/ui/android/res/drawable-mdpi/ic_launcher.png differ diff --git a/src/ui/android/res/drawable-mdpi/ic_mic.png b/src/ui/android/res/drawable-mdpi/ic_mic.png new file mode 100644 index 000000000..8f7f55cf9 Binary files /dev/null and b/src/ui/android/res/drawable-mdpi/ic_mic.png differ diff --git a/src/ui/android/res/drawable-xhdpi/ic_action_switch.png b/src/ui/android/res/drawable-xhdpi/ic_action_switch.png new file mode 100644 index 000000000..b5da00fb2 Binary files /dev/null and b/src/ui/android/res/drawable-xhdpi/ic_action_switch.png differ diff --git a/src/ui/android/res/drawable-xhdpi/ic_launcher.png b/src/ui/android/res/drawable-xhdpi/ic_launcher.png new file mode 100644 index 000000000..7f212cc6b Binary files /dev/null and b/src/ui/android/res/drawable-xhdpi/ic_launcher.png differ diff --git a/src/ui/android/res/drawable-xhdpi/ic_mic.png b/src/ui/android/res/drawable-xhdpi/ic_mic.png new file mode 100644 index 000000000..13d21274a Binary files /dev/null and b/src/ui/android/res/drawable-xhdpi/ic_mic.png differ diff --git a/src/ui/android/res/drawable-xxhdpi/ic_launcher.png b/src/ui/android/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 000000000..da2c7a235 Binary files /dev/null and b/src/ui/android/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/src/ui/android/res/drawable/brushed_metal.png b/src/ui/android/res/drawable/brushed_metal.png deleted file mode 100644 index c2f03fe7d..000000000 Binary files a/src/ui/android/res/drawable/brushed_metal.png and /dev/null differ diff --git a/src/ui/android/res/drawable/first_person_utterance_bg.xml b/src/ui/android/res/drawable/first_person_utterance_bg.xml new file mode 100644 index 000000000..9eb02aef1 --- /dev/null +++ b/src/ui/android/res/drawable/first_person_utterance_bg.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/src/ui/android/res/drawable/icon.png b/src/ui/android/res/drawable/icon.png deleted file mode 100644 index a07c69fa5..000000000 Binary files a/src/ui/android/res/drawable/icon.png and /dev/null differ diff --git a/src/ui/android/res/drawable/second_person_utterance_bg.xml b/src/ui/android/res/drawable/second_person_utterance_bg.xml new file mode 100644 index 000000000..4acf07c67 --- /dev/null +++ b/src/ui/android/res/drawable/second_person_utterance_bg.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/src/ui/android/res/layout/activity_main.xml b/src/ui/android/res/layout/activity_main.xml new file mode 100644 index 000000000..b0ccab0ea --- /dev/null +++ b/src/ui/android/res/layout/activity_main.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/ui/android/res/layout/first_person_utterance.xml b/src/ui/android/res/layout/first_person_utterance.xml new file mode 100644 index 000000000..55779ae8f --- /dev/null +++ b/src/ui/android/res/layout/first_person_utterance.xml @@ -0,0 +1,11 @@ + diff --git a/src/ui/android/res/layout/languages_item.xml b/src/ui/android/res/layout/languages_item.xml new file mode 100644 index 000000000..d5f47ab27 --- /dev/null +++ b/src/ui/android/res/layout/languages_item.xml @@ -0,0 +1,8 @@ + + diff --git a/src/ui/android/res/layout/second_person_utterance.xml b/src/ui/android/res/layout/second_person_utterance.xml new file mode 100644 index 000000000..416d85328 --- /dev/null +++ b/src/ui/android/res/layout/second_person_utterance.xml @@ -0,0 +1,12 @@ + + diff --git a/src/ui/android/res/values-sw600dp/dimens.xml b/src/ui/android/res/values-sw600dp/dimens.xml new file mode 100644 index 000000000..44f01db75 --- /dev/null +++ b/src/ui/android/res/values-sw600dp/dimens.xml @@ -0,0 +1,8 @@ + + + + + diff --git a/src/ui/android/res/values-sw720dp-land/dimens.xml b/src/ui/android/res/values-sw720dp-land/dimens.xml new file mode 100644 index 000000000..61e3fa8fb --- /dev/null +++ b/src/ui/android/res/values-sw720dp-land/dimens.xml @@ -0,0 +1,9 @@ + + + + 128dp + + diff --git a/src/ui/android/res/values-v11/styles.xml b/src/ui/android/res/values-v11/styles.xml new file mode 100644 index 000000000..3c02242ad --- /dev/null +++ b/src/ui/android/res/values-v11/styles.xml @@ -0,0 +1,11 @@ + + + + + + diff --git a/src/ui/android/res/values-v14/styles.xml b/src/ui/android/res/values-v14/styles.xml new file mode 100644 index 000000000..a91fd0372 --- /dev/null +++ b/src/ui/android/res/values-v14/styles.xml @@ -0,0 +1,12 @@ + + + + + + diff --git a/src/ui/android/res/values/dimens.xml b/src/ui/android/res/values/dimens.xml new file mode 100644 index 000000000..55c1e5908 --- /dev/null +++ b/src/ui/android/res/values/dimens.xml @@ -0,0 +1,7 @@ + + + + 16dp + 16dp + + diff --git a/src/ui/android/res/values/strings.xml b/src/ui/android/res/values/strings.xml index 0d7b9a606..26601be1b 100644 --- a/src/ui/android/res/values/strings.xml +++ b/src/ui/android/res/values/strings.xml @@ -1,8 +1,9 @@ - GFTranslator - Settings - Hello world! + GF Translator + + Microphone + Switch languages diff --git a/src/ui/android/res/values/styles.xml b/src/ui/android/res/values/styles.xml new file mode 100644 index 000000000..6ce89c7ba --- /dev/null +++ b/src/ui/android/res/values/styles.xml @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/ASR.java b/src/ui/android/src/org/grammaticalframework/ui/android/ASR.java new file mode 100644 index 000000000..ef6df5198 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/ASR.java @@ -0,0 +1,240 @@ + +package org.grammaticalframework.ui.android; + +import android.content.Context; +import android.content.Intent; +import android.os.Bundle; +import android.speech.RecognitionListener; +import android.speech.RecognizerIntent; +import android.speech.SpeechRecognizer; +import android.text.TextUtils; +import android.util.Log; + +import java.util.ArrayList; + +/** + * Convenience wrapper around the {@link SpeechRecognizer} API. + */ +public class ASR { + + private static final boolean DBG = true; + private static final String TAG = "ASR"; + + private final Context mContext; + + private SpeechRecognizer mSpeechRecognizer; + + private String mLanguage = null; + + private State mState = State.IDLE; + + private Listener mListener; + + public static enum State { + IDLE, INITIALIZING, WAITING_FOR_SPEECH, RECORDING, WAITING_FOR_RESULTS; + } + + public ASR(Context context) { + mContext = context; + if (SpeechRecognizer.isRecognitionAvailable(context)) { + mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(context); + mSpeechRecognizer.setRecognitionListener(new MyRecognitionListener()); + } + } + + public void setListener(Listener listener) { + mListener = listener; + } + + public void setLanguage(String language) { + mLanguage = language; + } + + public void startRecognition() { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + if (!TextUtils.isEmpty(mLanguage)) { + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, mLanguage); + } + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, + RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 2); + intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true); + // Weird, this shouldn't be required, but on ICS it seems to be + intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, + mContext.getPackageName()); + + mSpeechRecognizer.startListening(intent); + setState(State.INITIALIZING); + } + + public void stopRecognition() { + mSpeechRecognizer.stopListening(); + setState(State.IDLE); + } + + public boolean isRunning() { + return mState != State.IDLE; + } + + private void setState(State newState) { + if (DBG) Log.d(TAG, "Entering state: " + newState); + mState = newState; + if (mListener != null) { + mListener.onStateChanged(mState); + } + } + + public State getState() { + return mState; + } + + public void destroy() { + if (mSpeechRecognizer != null) { + mSpeechRecognizer.destroy(); + mSpeechRecognizer = null; + } + } + + private void handlePartialInput(String text) { + if (mListener != null) { + mListener.onPartialInput(text); + } + } + + private void handleSpeechInput(String text) { + if (mListener != null) { + mListener.onSpeechInput(text); + } + } + + private class MyRecognitionListener implements RecognitionListener { + @Override + public void onReadyForSpeech(Bundle params) { + if (DBG) Log.d(TAG, "onReadyForSpeech"); + setState(State.WAITING_FOR_SPEECH); + } + + @Override + public void onBeginningOfSpeech() { + if (DBG) Log.d(TAG, "onBeginningOfSpeech"); + setState(State.RECORDING); + } + + @Override + public void onBufferReceived(byte[] buffer) { + // Ignore + } + + @Override + public void onRmsChanged(float rmsdB) { + if (DBG) Log.d(TAG, "onRmsChanged(" + rmsdB + ")"); + } + + @Override + public void onEndOfSpeech() { + if (DBG) Log.d(TAG, "onEndOfSpeech"); + setState(State.WAITING_FOR_RESULTS); + } + + @Override + public void onError(int error) { + if (DBG) Log.d(TAG, "Error: " + errorMessage(error) + " (" + error + ")"); + setState(State.IDLE); + } + + private String errorMessage(int speechRecognizerError) { + switch(speechRecognizerError) { + case SpeechRecognizer.ERROR_NETWORK_TIMEOUT: + return "network timeout"; + case SpeechRecognizer.ERROR_NETWORK: + return "network"; + case SpeechRecognizer.ERROR_AUDIO: + return "audio"; + case SpeechRecognizer.ERROR_SERVER: + return "server"; + case SpeechRecognizer.ERROR_CLIENT: + return "client"; + case SpeechRecognizer.ERROR_SPEECH_TIMEOUT: + return "timeout waiting for speech"; + case SpeechRecognizer.ERROR_NO_MATCH: + return "no match found"; + case SpeechRecognizer.ERROR_RECOGNIZER_BUSY: + return "recognizer busy"; + case SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS: + return "insufficient permissions (missing RECORD_AUDIO?)"; + default: + return "unknown"; + } + } + + @Override + public void onEvent(int eventType, Bundle params) { + if (DBG) Log.d(TAG, "onEvent(" + eventType + ")"); + } + + @Override + public void onPartialResults(Bundle bundle) { + if (DBG) { + StringBuilder sb = new StringBuilder(); + sb.append("onPartialResults:"); + appendResults(sb, bundle); + Log.d(TAG, sb.toString()); + } + + String result = getResult(bundle); + if (!TextUtils.isEmpty(result)) { + handlePartialInput(result); + } + } + + @Override + public void onResults(Bundle bundle) { + if (DBG) { + StringBuilder sb = new StringBuilder(); + sb.append("onResults:"); + appendResults(sb, bundle); + Log.d(TAG, sb.toString()); + } + + setState(State.IDLE); + + String result = getResult(bundle); + if (!TextUtils.isEmpty(result)) { + handleSpeechInput(result); + } + } + + private String getResult(Bundle bundle) { + ArrayList results = + bundle.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION); + if (results != null && !results.isEmpty()) { + return results.get(0); + } else { + return null; + } + } + + private void appendResults(StringBuilder sb, Bundle bundle) { + ArrayList results = + bundle.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION); + float[] scores = bundle.getFloatArray(SpeechRecognizer.CONFIDENCE_SCORES); + + if (results != null) { + int size = results.size(); + for (int i = 0; i < size; i++) { + sb.append("\n> ").append(results.get(i)); + if (scores != null && i < scores.length) { + sb.append(" [").append(scores[i]).append("]"); + } + } + } + } + } + + public interface Listener { + void onPartialInput(String input); + void onSpeechInput(String input); + void onStateChanged(State newState); + } + +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/ConversationView.java b/src/ui/android/src/org/grammaticalframework/ui/android/ConversationView.java new file mode 100644 index 000000000..3923d13a0 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/ConversationView.java @@ -0,0 +1,62 @@ +package org.grammaticalframework.ui.android; + +import android.content.Context; +import android.util.AttributeSet; +import android.view.LayoutInflater; +import android.view.ViewGroup; +import android.widget.ScrollView; +import android.widget.TextView; + +public class ConversationView extends ScrollView { + + private LayoutInflater mInflater; + + private ViewGroup mContent; + + public ConversationView(Context context, AttributeSet attrs, int defStyle) { + super(context, attrs, defStyle); + } + + public ConversationView(Context context, AttributeSet attrs) { + super(context, attrs); + } + + public ConversationView(Context context) { + super(context); + } + + @Override + protected void onFinishInflate() { + super.onFinishInflate(); + mContent = (ViewGroup) findViewById(R.id.conversation_content); + mInflater = LayoutInflater.from(getContext()); + } + + public void addFirstPersonUtterance(CharSequence text) { + addUtterance(R.layout.first_person_utterance, text); + } + + public void addSecondPersonUtterance(CharSequence text) { + addUtterance(R.layout.second_person_utterance, text); + } + + private void addUtterance(int res, CharSequence text) { + TextView view = (TextView) mInflater.inflate(res, mContent, false); + view.setText(text); + mContent.addView(view); + post(new Runnable() { + public void run() { + fullScroll(FOCUS_DOWN); + } + }); + } + + public void updateLastUtterance(CharSequence text) { + int count = mContent.getChildCount(); + if (count > 0) { + TextView view = (TextView) mContent.getChildAt(count - 1); + view.setText(text); + } + } + +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/Language.java b/src/ui/android/src/org/grammaticalframework/ui/android/Language.java new file mode 100644 index 000000000..8adc74609 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/Language.java @@ -0,0 +1,31 @@ +package org.grammaticalframework.ui.android; + +public class Language { + private final String mLangCode; + private final String mLangName; + private final String mConcrete; + + public Language(String langCode, String langName, String concrete) { + mLangCode = langCode; + mLangName = langName; + mConcrete = concrete; + } + + public String getLangCode() { + return mLangCode; + } + + public String getLangName() { + return mLangName; + } + + String getConcrete() { + return mConcrete; + } + + @Override + public String toString() { + return getLangName(); + } + +} \ No newline at end of file diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/LanguageSelector.java b/src/ui/android/src/org/grammaticalframework/ui/android/LanguageSelector.java new file mode 100644 index 000000000..d3148cda4 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/LanguageSelector.java @@ -0,0 +1,51 @@ +package org.grammaticalframework.ui.android; + +import android.content.Context; +import android.util.AttributeSet; +import android.view.View; +import android.widget.AdapterView; +import android.widget.Spinner; + +import java.util.List; + +public class LanguageSelector extends Spinner { + + public LanguageSelector(Context context, AttributeSet attrs, int defStyle) { + super(context, attrs, defStyle); + } + + public LanguageSelector(Context context, AttributeSet attrs) { + super(context, attrs); + } + + public LanguageSelector(Context context) { + super(context); + } + + public void setLanguages(List languages) { + setAdapter(new LanguagesAdapter(getContext(), languages)); + } + + public void setSelectedLanguage(Language selected) { + setSelection(((LanguagesAdapter) getAdapter()).getPosition(selected)); + } + + public void setOnLanguageSelectedListener(final OnLanguageSelectedListener listener) { + setOnItemSelectedListener(new OnItemSelectedListener() { + @Override + public void onItemSelected(AdapterView parent, View view, int position, long id) { + if (listener != null) { + listener.onLanguageSelected((Language) parent.getItemAtPosition(position)); + } + } + @Override + public void onNothingSelected(AdapterView parent) { + } + }); + } + + public interface OnLanguageSelectedListener { + void onLanguageSelected(Language language); + } + +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/LanguagesAdapter.java b/src/ui/android/src/org/grammaticalframework/ui/android/LanguagesAdapter.java new file mode 100644 index 000000000..e39ed7bd9 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/LanguagesAdapter.java @@ -0,0 +1,16 @@ +package org.grammaticalframework.ui.android; + +import android.content.Context; +import android.widget.ArrayAdapter; +import android.widget.SpinnerAdapter; + + +import java.util.List; + +public class LanguagesAdapter extends ArrayAdapter implements SpinnerAdapter { + + public LanguagesAdapter(Context context, List objects) { + super(context, R.layout.languages_item, objects); + } + +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/LocaleUtils.java b/src/ui/android/src/org/grammaticalframework/ui/android/LocaleUtils.java new file mode 100644 index 000000000..9fb048908 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/LocaleUtils.java @@ -0,0 +1,42 @@ +package org.grammaticalframework.ui.android; + +import android.text.TextUtils; + +import java.util.Locale; + +/** + * Collections of utils to handle locales. + */ +public class LocaleUtils { + + /** + * Parses a locale string formatted by {@link Locale#toString()}. + * + * @return the parsed {@code Locale} or {@code defaultLocale} if the input was null or empty. + */ + public static Locale parseJavaLocale(String localeString, Locale defaultLocale) { + if (TextUtils.isEmpty(localeString)) { + return defaultLocale; + } + final char separator = '_'; + int pos1 = localeString.indexOf(separator); + if (pos1 == -1) { + return new Locale(localeString); + } + String language = localeString.substring(0, pos1); + + int start2 = pos1 + 1; + int pos2 = localeString.indexOf(separator, start2); + if (pos2 == -1) { + return new Locale(language, localeString.substring(start2)); + } + String country = localeString.substring(start2, pos2); + + int start3 = pos2 + 1; + int pos3 = localeString.indexOf(separator, start3); + String variant = (pos3 == -1) + ? localeString.substring(start3) + : localeString.substring(start3, pos3); + return new Locale(language, country, variant); + } +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/MainActivity.java b/src/ui/android/src/org/grammaticalframework/ui/android/MainActivity.java new file mode 100644 index 000000000..fba9987d6 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/MainActivity.java @@ -0,0 +1,202 @@ + +package org.grammaticalframework.ui.android; + +import android.app.Activity; +import android.os.AsyncTask; +import android.os.Bundle; +import android.speech.SpeechRecognizer; +import android.util.Log; +import android.view.View; +import android.view.View.OnClickListener; +import android.widget.ImageView; + +import org.grammaticalframework.ui.android.ASR.State; +import org.grammaticalframework.ui.android.LanguageSelector.OnLanguageSelectedListener; + +public class MainActivity extends Activity { + + private static final boolean DBG = true; + private static final String TAG = "DemoActivity"; + + private static final boolean FAKE_SPEECH = false; + + private ImageView mStartStopButton; + + private ConversationView mConversationView; + + private LanguageSelector mSourceLanguageView; + + private LanguageSelector mTargetLanguageView; + + private ImageView mSwitchLanguagesButton; + + private ASR mAsr; + + private TTS mTts; + + private Translator mTranslator; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + + mStartStopButton = (ImageView) findViewById(R.id.start_stop); + mConversationView = (ConversationView) findViewById(R.id.conversation); + mSourceLanguageView = (LanguageSelector) findViewById(R.id.source_language); + mTargetLanguageView = (LanguageSelector) findViewById(R.id.target_language); + mSwitchLanguagesButton = (ImageView) findViewById(R.id.switch_languages); + + mStartStopButton.setOnClickListener(new OnClickListener() { + @Override + public void onClick(View v) { + if (mAsr.isRunning()) { + stopRecognition(); + } else { + startRecognition(); + } + } + }); + + mStartStopButton.setEnabled(SpeechRecognizer.isRecognitionAvailable(this)); + + mAsr = new ASR(this); + mAsr.setListener(new SpeechInputListener()); + + mTts = new TTS(this); + + mTranslator = new Translator(this); + new AsyncTask() { + @Override + protected Void doInBackground(Void... params) { + mTranslator.init(); + return null; + } + }.execute(); + + mSourceLanguageView.setLanguages(mTranslator.getAvailableSourceLanguages()); + mSourceLanguageView.setSelectedLanguage(mTranslator.getSourceLanguage()); + mSourceLanguageView.setOnLanguageSelectedListener(new OnLanguageSelectedListener() { + @Override + public void onLanguageSelected(Language language) { + onSourceLanguageSelected(language); + } + }); + mTargetLanguageView.setLanguages(mTranslator.getAvailableTargetLanguages()); + mTargetLanguageView.setSelectedLanguage(mTranslator.getTargetLanguage()); + mTargetLanguageView.setOnLanguageSelectedListener(new OnLanguageSelectedListener() { + @Override + public void onLanguageSelected(Language language) { + onTargetLanguageSelected(language); + } + }); + + mSwitchLanguagesButton.setOnClickListener(new OnClickListener() { + @Override + public void onClick(View v) { + onSwitchLanguages(); + } + }); + } + + @Override + protected void onDestroy() { + if (mAsr != null) { + mAsr.destroy(); + mAsr = null; + } + if (mTts != null) { + mTts.destroy(); + mTts = null; + } + super.onDestroy(); + } + + void onSourceLanguageSelected(Language language) { + mTranslator.setSourceLanguage(language); + } + + void onTargetLanguageSelected(Language language) { + mTranslator.setTargetLanguage(language); + } + + public String getSourceLanguageCode() { + return mTranslator.getSourceLanguage().getLangCode(); + } + + public String getTargetLanguageCode() { + return mTranslator.getTargetLanguage().getLangCode(); + } + + void onSwitchLanguages() { + Language newSource = mTranslator.getTargetLanguage(); + Language newTarget = mTranslator.getSourceLanguage(); + mSourceLanguageView.setSelectedLanguage(newSource); + mTargetLanguageView.setSelectedLanguage(newTarget); + } + + private void startRecognition() { + mConversationView.addFirstPersonUtterance("..."); + + if (FAKE_SPEECH) { + handleSpeechInput("where is the hotel"); + } else { + mAsr.setLanguage(getSourceLanguageCode()); + mAsr.startRecognition(); + } + } + + private void stopRecognition() { + mAsr.stopRecognition(); + } + + private void handlePartialSpeechInput(String input) { + mConversationView.updateLastUtterance(input); + } + + private void handleSpeechInput(final String input) { + mConversationView.updateLastUtterance(input); + new AsyncTask() { + @Override + protected String doInBackground(Void... params) { + return mTranslator.translate(input); + } + + @Override + protected void onPostExecute(String result) { + outputText(result); + } + }.execute(); + } + + private void outputText(String text) { + if (DBG) Log.d(TAG, "Speaking: " + text); + mConversationView.addSecondPersonUtterance(text); + if (!FAKE_SPEECH) { + mTts.setLanguage(getTargetLanguageCode()); + mTts.speak(text); + } + } + + private class SpeechInputListener implements ASR.Listener { + + @Override + public void onPartialInput(String input) { + handlePartialSpeechInput(input); + } + + @Override + public void onSpeechInput(String input) { + handleSpeechInput(input); + } + + @Override + public void onStateChanged(State newState) { +// if (newState == ASR.State.IDLE) { +// mStartStopButton.setImageResource(R.drawable.mic_idle); +// } else { +// mStartStopButton.setImageResource(R.drawable.mic_open); +// } + } + } +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/TTS.java b/src/ui/android/src/org/grammaticalframework/ui/android/TTS.java new file mode 100644 index 000000000..6993a3fc6 --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/TTS.java @@ -0,0 +1,63 @@ +package org.grammaticalframework.ui.android; + +import android.content.Context; +import android.speech.tts.TextToSpeech; +import android.util.Log; + +import java.util.HashMap; +import java.util.Locale; + +public class TTS { + + private static final String TAG = "TTS"; + + private TextToSpeech mTts; + + public TTS(Context context) { + mTts = new TextToSpeech(context, new InitListener()); + } + + public void setLanguage(String language) { + Locale locale = LocaleUtils.parseJavaLocale(language.replace('-', '_'), + Locale.getDefault()); + + int result = mTts.setLanguage(locale); + if (result == TextToSpeech.LANG_MISSING_DATA || + result == TextToSpeech.LANG_NOT_SUPPORTED) { + Log.e(TAG, "Language is not available"); + } else { + // TODO: the language may be available for the locale, + // but not for the specified country and variant. + } + } + + // TODO: handle speak() calls before service connects + public void speak(String text) { + HashMap params = new HashMap(); + // TODO: how can I get network / embedded fallback? + // Using both crashes the TTS engine if the offline data is not installed + // Using only one doesn't allow the other +// params.put(TextToSpeech.Engine.KEY_FEATURE_NETWORK_SYNTHESIS, "true"); +// params.put(TextToSpeech.Engine.KEY_FEATURE_EMBEDDED_SYNTHESIS, "true"); + mTts.speak(text, TextToSpeech.QUEUE_FLUSH, params); + } + + public void destroy() { + if (mTts != null) { + mTts.stop(); + mTts.shutdown(); + } + } + + private class InitListener implements TextToSpeech.OnInitListener { + @Override + public void onInit(int status) { + if (status == TextToSpeech.SUCCESS) { + Log.d(TAG, "Initialized TTS"); + } else { + Log.e(TAG, "Failed to initialize TTS"); + } + } + + } +} diff --git a/src/ui/android/src/org/grammaticalframework/ui/android/Translator.java b/src/ui/android/src/org/grammaticalframework/ui/android/Translator.java new file mode 100644 index 000000000..9ecdb104e --- /dev/null +++ b/src/ui/android/src/org/grammaticalframework/ui/android/Translator.java @@ -0,0 +1,144 @@ +package org.grammaticalframework.ui.android; + +import android.content.Context; +import android.util.Log; + +import org.grammaticalframework.pgf.Concr; +import org.grammaticalframework.pgf.Expr; +import org.grammaticalframework.pgf.PGF; +import org.grammaticalframework.pgf.ParseError; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; + +public class Translator { + + private static final String TAG = "Translator"; + + // TODO: allow changing + private String mGrammar = "ResourceDemo.pgf"; + + // TODO: build dynamically? + private Language[] mLanguages = { + new Language("en-US", "English", "ResourceDemoEng"), + new Language("de-DE", "German", "ResourceDemoGer"), + new Language("es-ES", "Spanish", "ResourceDemoSpa"), + new Language("fr-FR", "French", "ResourceDemoFre"), + }; + + private final Context mContext; + + private Language mSourceLanguage; + + private Language mTargetLanguage; + + private PGF mPgf; + + public Translator(Context context) { + mContext = context; + } + + public Context getContext() { + return mContext; + } + + public List getAvailableSourceLanguages() { + return Arrays.asList(mLanguages); + } + + public List getAvailableTargetLanguages() { + return Arrays.asList(mLanguages); + } + + public void setSourceLanguage(Language language) { + mSourceLanguage = language; + } + + public void setTargetLanguage(Language language) { + mTargetLanguage = language; + } + + public Language getSourceLanguage() { + return mSourceLanguage != null ? mSourceLanguage : mLanguages[0]; + } + + public Language getTargetLanguage() { + return mTargetLanguage != null ? mTargetLanguage : mLanguages[1]; + } + + /** + * Takes a lot of time. Must not be called on the main thread. + */ + public void init() { + ensureLoaded(mGrammar); + } + + /** + * Takes a lot of time. Must not be called on the main thread. + */ + public String translate(String input) { + ensureLoaded(mGrammar); + return translateInternal(input); + } + + private synchronized void ensureLoaded(String grammarName) { + if (mPgf != null) return; + + try { + // TODO: use PGF API to read this directly from assets + Log.d(TAG, "Copying grammar..."); + File file = copyAsset(grammarName); + Log.d(TAG, "Trying to open " + file); + mPgf = PGF.readPGF(file.getPath()); + } catch (FileNotFoundException e) { + Log.e(TAG, "File not found", e); + } catch (IOException e) { + Log.e(TAG, "Error loading grammar", e); + } + } + + private File copyAsset(String asset) throws IOException { + InputStream in = null; + OutputStream out = null; + try { + in = getContext().getAssets().open(asset); + out = getContext().openFileOutput(asset, Context.MODE_PRIVATE); + byte[] buf = new byte[4096]; + int len; + while ((len = in.read(buf)) > 0) { + out.write(buf, 0, len); + } + return getContext().getFileStreamPath(asset); + } finally { + if (in != null) { + in.close(); + } + if (out != null) { + out.close(); + } + } + } + + protected String translateInternal(String input) { + try { + Concr sourceGrammar = getConcr(getSourceLanguage().getConcrete()); + Expr expr = sourceGrammar.parseBest("S", input); + Concr targetGrammar = getConcr(getTargetLanguage().getConcrete()); + String output = targetGrammar.linearize(expr); + return output; + } catch (ParseError e) { + Log.e(TAG, "Parse error: " + e); + return "parse error"; // TODO: no no no + } + } + + private Concr getConcr(String name) { + return mPgf.getLanguages().get(name); + } + +} diff --git a/src/ui/android/src/se/fnord/android/layout/PredicateLayout.java b/src/ui/android/src/se/fnord/android/layout/PredicateLayout.java deleted file mode 100644 index 4734d4618..000000000 --- a/src/ui/android/src/se/fnord/android/layout/PredicateLayout.java +++ /dev/null @@ -1,134 +0,0 @@ -package se.fnord.android.layout; - -import android.content.Context; -import android.util.AttributeSet; -import android.view.View; -import android.view.ViewGroup; - -/** - * ViewGroup that arranges child views in a similar way to text, with them laid - * out one line at a time and "wrapping" to the next line as needed. - * - * Code licensed under CC-by-SA - * - * @author Henrik Gustafsson - * @see http://stackoverflow.com/questions/549451/line-breaking-widget-layout-for-android - * @license http://creativecommons.org/licenses/by-sa/2.5/ - * - */ -public class PredicateLayout extends ViewGroup { - - private int line_height; - - public static class LayoutParams extends ViewGroup.LayoutParams { - public final int horizontal_spacing; - public final int vertical_spacing; - - /** - * @param horizontal_spacing Pixels between items, horizontally - * @param vertical_spacing Pixels between items, vertically - */ - public LayoutParams(int horizontal_spacing, int vertical_spacing) { - this(0, 0, horizontal_spacing, vertical_spacing); - } - - /** - * @param width - * @param height - * @param horizontal_spacing Pixels between items, horizontally - * @param vertical_spacing Pixels between items, vertically - */ - public LayoutParams(int width, int height, int horizontal_spacing, int vertical_spacing) { - super(width, height); - this.horizontal_spacing = horizontal_spacing; - this.vertical_spacing = vertical_spacing; - } - } - - public PredicateLayout(Context context) { - super(context); - } - - public PredicateLayout(Context context, AttributeSet attrs){ - super(context, attrs); - } - - @Override - protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { - assert(MeasureSpec.getMode(widthMeasureSpec) != MeasureSpec.UNSPECIFIED); - - final int width = MeasureSpec.getSize(widthMeasureSpec) - getPaddingLeft() - getPaddingRight(); - int height = MeasureSpec.getSize(heightMeasureSpec) - getPaddingTop() - getPaddingBottom(); - final int count = getChildCount(); - int line_height = 0; - - int xpos = getPaddingLeft(); - int ypos = getPaddingTop(); - - for (int i = 0; i < count; i++) { - final View child = getChildAt(i); - if (child.getVisibility() != GONE) { - final LayoutParams lp = (LayoutParams) child.getLayoutParams(); - child.measure( - MeasureSpec.makeMeasureSpec(width, MeasureSpec.AT_MOST), - MeasureSpec.makeMeasureSpec(height, MeasureSpec.AT_MOST)); - - final int childw = child.getMeasuredWidth(); - line_height = Math.max(line_height, child.getMeasuredHeight() + lp.vertical_spacing); - - if (xpos + childw > width) { - xpos = getPaddingLeft(); - ypos += line_height; - } - - xpos += childw + lp.horizontal_spacing; - } - } - this.line_height = line_height; - - if (MeasureSpec.getMode(heightMeasureSpec) == MeasureSpec.UNSPECIFIED){ - height = ypos + line_height; - - } else if (MeasureSpec.getMode(heightMeasureSpec) == MeasureSpec.AT_MOST){ - if (ypos + line_height < height){ - height = ypos + line_height; - } - } - setMeasuredDimension(width, height); - } - - @Override - protected ViewGroup.LayoutParams generateDefaultLayoutParams() { - return new LayoutParams(1, 1); // default of 1px spacing - } - - @Override - protected boolean checkLayoutParams(ViewGroup.LayoutParams p) { - if (p instanceof LayoutParams) - return true; - return false; - } - - @Override - protected void onLayout(boolean changed, int l, int t, int r, int b) { - final int count = getChildCount(); - final int width = r - l; - int xpos = getPaddingLeft(); - int ypos = getPaddingTop(); - - for (int i = 0; i < count; i++) { - final View child = getChildAt(i); - if (child.getVisibility() != GONE) { - final int childw = child.getMeasuredWidth(); - final int childh = child.getMeasuredHeight(); - final LayoutParams lp = (LayoutParams) child.getLayoutParams(); - if (xpos + childw > width) { - xpos = getPaddingLeft(); - ypos += line_height; - } - child.layout(xpos, ypos, xpos + childw, ypos + childh); - xpos += childw + lp.horizontal_spacing; - } - } - } -} \ No newline at end of file