From a49dd7157ee56e5ff8d05d29416f44403b241a46 Mon Sep 17 00:00:00 2001 From: Noah Negrey Date: Thu, 19 Sep 2019 15:44:28 -0600 Subject: [PATCH] Dialogflow sample (#114) * Initial commit * Addressed review comments * Added README File * Update README.md * Update README.md * Update README.md * refactor: refactoring the code * Add stopwatchagent.zip file * Update README.md * Update README.md * Changes has per Github comments * Combine the three requests into one. Dropdown instead of multi-select checkbox. Remove the Readback on the user input * Addressed review comments * Using the Audio returned from Dialogflow. * Addressed review comments * Separated the Authentication code from AppController.java to AuthUtils.java * Code Refinement * Update README.md * Update README.md * Addressed review comments * function to convert the time from UTC to local TimeZone added * Code readability update * Update README.md index.js Link update * Speech2speech Initial commit * Separate samples from single PR into multiple * Update gcp_project_id value * Add missing steps / small code cleanup * Nit style * Add support to detect intents via audio --- .../examples/dialogflow/ui/ChatActivity.java | 140 +++++++++++------- .../examples/dialogflow/utils/ApiRequest.java | 66 ++++++--- dialogflow/app/src/main/res/values/colors.xml | 2 +- 3 files changed, 134 insertions(+), 74 deletions(-) diff --git a/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/ui/ChatActivity.java b/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/ui/ChatActivity.java index 71cc26cb..bd127306 100644 --- a/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/ui/ChatActivity.java +++ b/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/ui/ChatActivity.java @@ -17,15 +17,15 @@ package com.google.cloud.examples.dialogflow.ui; import android.Manifest; -import android.content.ActivityNotFoundException; import android.content.BroadcastReceiver; import android.content.Context; +import android.content.DialogInterface; import android.content.Intent; import android.content.IntentFilter; +import android.media.MediaRecorder; import android.os.AsyncTask; import android.os.Bundle; import android.os.Handler; -import android.speech.RecognizerIntent; import android.text.TextUtils; import android.view.MenuItem; import android.view.View; @@ -47,8 +47,13 @@ import com.google.cloud.examples.dialogflow.utils.ApiRequest; import com.google.cloud.examples.dialogflow.utils.AuthUtils; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.Date; +import java.util.Objects; public class ChatActivity extends AppCompatActivity { @@ -67,8 +72,8 @@ public class ChatActivity extends AppCompatActivity { private boolean tts = false; private boolean knowledge = false; private boolean sentiment = false; - - private String voiceInput = ""; + private boolean audioRequest = false; + private String fileName; /** * Broadcast receiver to hide the progress dialog when token is received @@ -77,9 +82,10 @@ public class ChatActivity extends AppCompatActivity { @Override public void onReceive(Context context, Intent intent) { alert.dismiss(); - if (!voiceInput.equals("")) { - sendMsg(voiceInput); - } else if (!etMsg.getText().toString().trim().equals("")) { + if (audioRequest) { + audioRequest = false; + sendAudio(); + } else if (!etMsg.getText().toString().trim().isEmpty()) { sendMsg(etMsg.getText().toString().trim()); } } @@ -143,9 +149,8 @@ protected void onCreate(Bundle savedInstanceState) { initViews(); setupRecyclerView(); initListeners(); - + fileName = Objects.requireNonNull(getExternalCacheDir()).getAbsolutePath() + "/temp.raw"; apiRequest = new ApiRequest(); - } /** @@ -203,20 +208,13 @@ public void onClick(View v) { }); } - - /** - * function to send the message - * - * @param msg : message sent from user - */ private void sendMsg(String msg) { if (!TextUtils.isEmpty(msg)) { // check if the token is received and expiry time is received and not expired if (AuthUtils.isTokenValid()) { addMsg(msg, 1); etMsg.setText(""); - voiceInput = ""; - new APIRequest(AuthUtils.token, AuthUtils.expiryTime, msg, tts, sentiment, + new APIRequest(AuthUtils.token, AuthUtils.expiryTime, msg, null, tts, sentiment, knowledge).execute(); } else { // get new token if expired or not received @@ -228,54 +226,79 @@ private void sendMsg(String msg) { } } + private void sendAudio() { + File file = new File(fileName); + int size = (int) file.length(); + byte[] audioBytes = new byte[size]; + try { + BufferedInputStream buf = new BufferedInputStream(new FileInputStream(file)); + buf.read(audioBytes, 0, audioBytes.length); + buf.close(); + } catch (Exception e) { + e.printStackTrace(); + return; + } + + // check if the token is expired or if we have not yet requested one + if (AuthUtils.isTokenValid()) { + etMsg.setText(""); + audioRequest = false; + new APIRequest(AuthUtils.token, AuthUtils.expiryTime, null, audioBytes, tts, sentiment, + knowledge).execute(); + } else { + // get new token if expired or not received + audioRequest = true; + getNewToken(); + } + } + private void getNewToken() { showProgressDialog(); AuthUtils.callFirebaseFunction(); } private void promptSpeechInput() { - Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); - intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, - RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); - intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1000); - intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, "en-US"); - intent.putExtra(RecognizerIntent.EXTRA_PROMPT, - "Speak"); + final MediaRecorder recorder = new MediaRecorder(); + recorder.setAudioSource(MediaRecorder.AudioSource.MIC); + recorder.setOutputFormat(MediaRecorder.OutputFormat.AMR_NB); + recorder.setAudioSamplingRate(8000); + recorder.setAudioEncoder(MediaRecorder.AudioEncoder.DEFAULT); + recorder.setOutputFile(fileName); + try { - startActivityForResult(intent, 101); - } catch (ActivityNotFoundException a) { - Toast.makeText(getApplicationContext(), - "Not Supported", + recorder.prepare(); + } catch (IOException e) { + Toast.makeText( + getApplicationContext(), + "Failed to record audio", Toast.LENGTH_SHORT).show(); + return; } - } - @Override - protected void onActivityResult(int requestCode, int resultCode, Intent data) { - super.onActivityResult(requestCode, resultCode, data); - if (resultCode == RESULT_OK) { - if (requestCode == 101) { - ArrayList result = data - .getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS); - voiceInput = result.get(0); - sendMsg(result.get(0)); - } - } + AlertDialog alertDialog = new AlertDialog.Builder(this) + .setMessage("Recording") + .setPositiveButton("Send", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialogInterface, int i) { + recorder.stop(); + recorder.release(); + sendAudio(); + } + }) + .create(); + + recorder.start(); + alertDialog.show(); } public void checkPermissions() { - ArrayList arrPerm = new ArrayList<>(); - arrPerm.add(Manifest.permission.INTERNET); - arrPerm.add(Manifest.permission.RECORD_AUDIO); - arrPerm.add(Manifest.permission.READ_EXTERNAL_STORAGE); - arrPerm.add(Manifest.permission.WRITE_EXTERNAL_STORAGE); - + String[] permissions = new String[4]; + permissions[0] = Manifest.permission.INTERNET; + permissions[1] = Manifest.permission.RECORD_AUDIO; + permissions[2] = Manifest.permission.READ_EXTERNAL_STORAGE; + permissions[3] = Manifest.permission.WRITE_EXTERNAL_STORAGE; - if (!arrPerm.isEmpty()) { - String[] permissions = new String[arrPerm.size()]; - permissions = arrPerm.toArray(permissions); - ActivityCompat.requestPermissions(this, permissions, 1); - } + ActivityCompat.requestPermissions(this, permissions, 1); } @Override @@ -315,7 +338,6 @@ public void showMorePopup() { @Override public boolean onMenuItemClick(MenuItem item) { - switch (item.getItemId()) { case R.id.action_tts: tts = !tts; @@ -342,15 +364,17 @@ private class APIRequest extends AsyncTask { private String token; private Date expiryTime; private String msg; + private byte[] audioBytes; private boolean tts; private boolean sentiment; private boolean knowledge; - public APIRequest(String token, Date expiryTime, String msg, boolean tts, boolean sentiment, + APIRequest(String token, Date expiryTime, String msg, byte[] audioBytes, boolean tts, boolean sentiment, boolean knowledge) { this.token = token; this.expiryTime = expiryTime; this.msg = msg; + this.audioBytes = audioBytes; this.tts = tts; this.sentiment = sentiment; this.knowledge = knowledge; @@ -358,13 +382,19 @@ public APIRequest(String token, Date expiryTime, String msg, boolean tts, boolea @Override protected String doInBackground(Void... voids) { - return apiRequest.callAPI(token, expiryTime, msg, tts, sentiment, knowledge); + return apiRequest.callAPI(token, expiryTime, msg, audioBytes, tts, sentiment, knowledge); } @Override protected void onPostExecute(String response) { super.onPostExecute(response); - addMsg(response, 0); + if (audioBytes != null) { + int index = response.indexOf("|"); + addMsg(response.substring(index+1), 1); + addMsg(response.substring(0, index), 0); + } else { + addMsg(response, 0); + } } } diff --git a/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/utils/ApiRequest.java b/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/utils/ApiRequest.java index 1add1efc..b68545e4 100644 --- a/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/utils/ApiRequest.java +++ b/dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/utils/ApiRequest.java @@ -20,6 +20,8 @@ import com.google.auth.Credentials; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.dialogflow.v2beta1.AudioEncoding; +import com.google.cloud.dialogflow.v2beta1.InputAudioConfig; import com.google.cloud.dialogflow.v2beta1.DetectIntentRequest; import com.google.cloud.dialogflow.v2beta1.DetectIntentResponse; import com.google.cloud.dialogflow.v2beta1.KnowledgeAnswers; @@ -35,6 +37,7 @@ import com.google.cloud.dialogflow.v2beta1.SessionsSettings; import com.google.cloud.dialogflow.v2beta1.TextInput; import com.google.cloud.examples.dialogflow.AppController; +import com.google.protobuf.ByteString; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -64,23 +67,26 @@ public ApiRequest() { * @param knowledge : send message to knowledge base if true * @return : response from the server */ - public String callAPI(String accessToken, Date expiryTime, String msg, boolean tts, + public String callAPI(String accessToken, Date expiryTime, String msg, byte[] audioBytes, boolean tts, boolean sentiment, boolean knowledge) { this.token = accessToken; this.tokenExpiration = expiryTime; - return detectIntent(msg, tts, sentiment, knowledge); + + return detectIntent(msg, audioBytes, tts, sentiment, knowledge); } /** * function to getting the results from the dialogflow * - * @param msg : message sent by the user - * @param tts : send message to text to speech if true - * @param sentiment : send message to sentiment analysis if true - * @param knowledge : send message to knowledge base if true + * @param msg : message sent by the user + * @param audioBytes : audio sent by the user + * @param tts : send message to text to speech if true + * @param sentiment : send message to sentiment analysis if true + * @param knowledge : send message to knowledge base if true * @return : response from the server */ - private String detectIntent(String msg, boolean tts, boolean sentiment, boolean knowledge) { + private String detectIntent(String msg, byte[] audioBytes, boolean tts, boolean sentiment, + boolean knowledge) { try { AccessToken accessToken = new AccessToken(token, tokenExpiration); Credentials credentials = GoogleCredentials.create(accessToken); @@ -92,17 +98,30 @@ private String detectIntent(String msg, boolean tts, boolean sentiment, boolean SessionName sessionName = SessionName.of(AppController.PROJECT_ID, AppController.SESSION_ID); - // Set the text (hello) and language code (en-US) for the query - TextInput textInput = TextInput.newBuilder() - .setText(msg) - .setLanguageCode("en-US") - .build(); - - // Build the query with the TextInput - QueryInput queryInput = QueryInput.newBuilder().setText(textInput).build(); + QueryInput queryInput; + if (msg != null) { + // Set the text (hello) and language code (en-US) for the query + TextInput textInput = TextInput.newBuilder() + .setText(msg) + .setLanguageCode("en-US") + .build(); + + // Build the query with the TextInput + queryInput = QueryInput.newBuilder().setText(textInput).build(); + } else { + // Instructs the speech recognizer how to process the audio content. + InputAudioConfig inputAudioConfig = InputAudioConfig.newBuilder() + .setAudioEncoding(AudioEncoding.AUDIO_ENCODING_AMR) + .setLanguageCode("en-US") + .setSampleRateHertz(8000) + .build(); + + // Build the query with the TextInput + queryInput = QueryInput.newBuilder().setAudioConfig(inputAudioConfig).build(); + } DetectIntentRequest detectIntentRequest = getDetectIntentRequest(sessionName, - queryInput, tts, sentiment, knowledge, fixedCredentialsProvider); + queryInput, tts, sentiment, knowledge, fixedCredentialsProvider, audioBytes); DetectIntentResponse detectIntentResponse = sessionsClient.detectIntent(detectIntentRequest); @@ -113,7 +132,14 @@ private String detectIntent(String msg, boolean tts, boolean sentiment, boolean AppController.playAudio(detectIntentResponse.getOutputAudio().toByteArray()); } - return handleResults(detectIntentResponse); + if (msg != null) { + return handleResults(detectIntentResponse); + } else { + return String.format( + "%s|%s", + handleResults(detectIntentResponse), + detectIntentResponse.getQueryResult().getQueryText()); + } } catch (Exception ex) { ex.printStackTrace(); return ex.getMessage(); @@ -133,12 +159,16 @@ private String detectIntent(String msg, boolean tts, boolean sentiment, boolean */ private DetectIntentRequest getDetectIntentRequest(SessionName sessionName, QueryInput queryInput, boolean tts, boolean sentiment, boolean knowledge, - FixedCredentialsProvider fixedCredentialsProvider) throws Exception { + FixedCredentialsProvider fixedCredentialsProvider, byte[] audioBytes) throws Exception { DetectIntentRequest.Builder detectIntentRequestBuilder = DetectIntentRequest.newBuilder() .setSession(sessionName.toString()) .setQueryInput(queryInput); + if (audioBytes != null) { + detectIntentRequestBuilder.setInputAudio(ByteString.copyFrom(audioBytes)); + } + QueryParameters.Builder queryParametersBuilder = QueryParameters.newBuilder(); if (tts) { diff --git a/dialogflow/app/src/main/res/values/colors.xml b/dialogflow/app/src/main/res/values/colors.xml index 06feb4ac..6fcd3fb4 100644 --- a/dialogflow/app/src/main/res/values/colors.xml +++ b/dialogflow/app/src/main/res/values/colors.xml @@ -2,6 +2,6 @@ #ff9800 #ef6c00 - #FF4081 + #ff9800 #FFFFFF