Skip to content

Commit

Permalink
Dialogflow sample (GoogleCloudPlatform#114)
Browse files Browse the repository at this point in the history 
* Initial commit

* Addressed review comments

* Added README File

* Update README.md

* Update README.md

* Update README.md

* refactor: refactoring the code

* Add stopwatchagent.zip file

* Update README.md

* Update README.md

* Changes has per Github comments

*  Combine the three requests into one. Dropdown instead of multi-select checkbox. Remove the Readback on the user input

* Addressed review comments

* Using the Audio returned from Dialogflow.

* Addressed review comments

* Separated the Authentication code from AppController.java to AuthUtils.java

* Code Refinement

* Update README.md

* Update README.md

* Addressed review comments

* function to convert the time from UTC to local TimeZone added

* Code readability update

* Update README.md

index.js Link update

* Speech2speech Initial commit

* Separate samples from single PR into multiple

* Update gcp_project_id value

* Add missing steps / small code cleanup

* Nit style

* Add support to detect intents via audio
  • Loading branch information
@nnegrey
nnegrey authored Sep 19, 2019
1 parent 4634133 commit a49dd71
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 74 deletions.
140 changes: 85 additions & 55 deletions dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/ui/ChatActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@
package com.google.cloud.examples.dialogflow.ui;

import android.Manifest;
import android.content.ActivityNotFoundException;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.IntentFilter;
import android.media.MediaRecorder;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.Handler;
import android.speech.RecognizerIntent;
import android.text.TextUtils;
import android.view.MenuItem;
import android.view.View;
Expand All @@ -47,8 +47,13 @@
import com.google.cloud.examples.dialogflow.utils.ApiRequest;
import com.google.cloud.examples.dialogflow.utils.AuthUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Objects;

public class ChatActivity extends AppCompatActivity {

Expand All @@ -67,8 +72,8 @@ public class ChatActivity extends AppCompatActivity {
private boolean tts = false;
private boolean knowledge = false;
private boolean sentiment = false;

private String voiceInput = "";
private boolean audioRequest = false;
private String fileName;

/**
* Broadcast receiver to hide the progress dialog when token is received
Expand All @@ -77,9 +82,10 @@ public class ChatActivity extends AppCompatActivity {
@Override
public void onReceive(Context context, Intent intent) {
alert.dismiss();
if (!voiceInput.equals("")) {
sendMsg(voiceInput);
} else if (!etMsg.getText().toString().trim().equals("")) {
if (audioRequest) {
audioRequest = false;
sendAudio();
} else if (!etMsg.getText().toString().trim().isEmpty()) {
sendMsg(etMsg.getText().toString().trim());
}
}
Expand Down Expand Up @@ -143,9 +149,8 @@ protected void onCreate(Bundle savedInstanceState) {
initViews();
setupRecyclerView();
initListeners();

fileName = Objects.requireNonNull(getExternalCacheDir()).getAbsolutePath() + "/temp.raw";
apiRequest = new ApiRequest();

}

/**
Expand Down Expand Up @@ -203,20 +208,13 @@ public void onClick(View v) {
});
}


/**
* function to send the message
*
* @param msg : message sent from user
*/
private void sendMsg(String msg) {
if (!TextUtils.isEmpty(msg)) {
// check if the token is received and expiry time is received and not expired
if (AuthUtils.isTokenValid()) {
addMsg(msg, 1);
etMsg.setText("");
voiceInput = "";
new APIRequest(AuthUtils.token, AuthUtils.expiryTime, msg, tts, sentiment,
new APIRequest(AuthUtils.token, AuthUtils.expiryTime, msg, null, tts, sentiment,
knowledge).execute();
} else {
// get new token if expired or not received
Expand All @@ -228,54 +226,79 @@ private void sendMsg(String msg) {
}
}

private void sendAudio() {
File file = new File(fileName);
int size = (int) file.length();
byte[] audioBytes = new byte[size];
try {
BufferedInputStream buf = new BufferedInputStream(new FileInputStream(file));
buf.read(audioBytes, 0, audioBytes.length);
buf.close();
} catch (Exception e) {
e.printStackTrace();
return;
}

// check if the token is expired or if we have not yet requested one
if (AuthUtils.isTokenValid()) {
etMsg.setText("");
audioRequest = false;
new APIRequest(AuthUtils.token, AuthUtils.expiryTime, null, audioBytes, tts, sentiment,
knowledge).execute();
} else {
// get new token if expired or not received
audioRequest = true;
getNewToken();
}
}

private void getNewToken() {
showProgressDialog();
AuthUtils.callFirebaseFunction();
}

private void promptSpeechInput() {
Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,
RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1000);
intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, "en-US");
intent.putExtra(RecognizerIntent.EXTRA_PROMPT,
"Speak");
final MediaRecorder recorder = new MediaRecorder();
recorder.setAudioSource(MediaRecorder.AudioSource.MIC);
recorder.setOutputFormat(MediaRecorder.OutputFormat.AMR_NB);
recorder.setAudioSamplingRate(8000);
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.DEFAULT);
recorder.setOutputFile(fileName);

try {
startActivityForResult(intent, 101);
} catch (ActivityNotFoundException a) {
Toast.makeText(getApplicationContext(),
"Not Supported",
recorder.prepare();
} catch (IOException e) {
Toast.makeText(
getApplicationContext(),
"Failed to record audio",
Toast.LENGTH_SHORT).show();
return;
}
}

@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
super.onActivityResult(requestCode, resultCode, data);
if (resultCode == RESULT_OK) {
if (requestCode == 101) {
ArrayList<String> result = data
.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
voiceInput = result.get(0);
sendMsg(result.get(0));
}
}
AlertDialog alertDialog = new AlertDialog.Builder(this)
.setMessage("Recording")
.setPositiveButton("Send", new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialogInterface, int i) {
recorder.stop();
recorder.release();
sendAudio();
}
})
.create();

recorder.start();
alertDialog.show();
}

public void checkPermissions() {
ArrayList<String> arrPerm = new ArrayList<>();
arrPerm.add(Manifest.permission.INTERNET);
arrPerm.add(Manifest.permission.RECORD_AUDIO);
arrPerm.add(Manifest.permission.READ_EXTERNAL_STORAGE);
arrPerm.add(Manifest.permission.WRITE_EXTERNAL_STORAGE);

String[] permissions = new String[4];
permissions[0] = Manifest.permission.INTERNET;
permissions[1] = Manifest.permission.RECORD_AUDIO;
permissions[2] = Manifest.permission.READ_EXTERNAL_STORAGE;
permissions[3] = Manifest.permission.WRITE_EXTERNAL_STORAGE;

if (!arrPerm.isEmpty()) {
String[] permissions = new String[arrPerm.size()];
permissions = arrPerm.toArray(permissions);
ActivityCompat.requestPermissions(this, permissions, 1);
}
ActivityCompat.requestPermissions(this, permissions, 1);
}

@Override
Expand Down Expand Up @@ -315,7 +338,6 @@ public void showMorePopup() {

@Override
public boolean onMenuItemClick(MenuItem item) {

switch (item.getItemId()) {
case R.id.action_tts:
tts = !tts;
Expand All @@ -342,29 +364,37 @@ private class APIRequest extends AsyncTask<Void, Void, String> {
private String token;
private Date expiryTime;
private String msg;
private byte[] audioBytes;
private boolean tts;
private boolean sentiment;
private boolean knowledge;

public APIRequest(String token, Date expiryTime, String msg, boolean tts, boolean sentiment,
APIRequest(String token, Date expiryTime, String msg, byte[] audioBytes, boolean tts, boolean sentiment,
boolean knowledge) {
this.token = token;
this.expiryTime = expiryTime;
this.msg = msg;
this.audioBytes = audioBytes;
this.tts = tts;
this.sentiment = sentiment;
this.knowledge = knowledge;
}

@Override
protected String doInBackground(Void... voids) {
return apiRequest.callAPI(token, expiryTime, msg, tts, sentiment, knowledge);
return apiRequest.callAPI(token, expiryTime, msg, audioBytes, tts, sentiment, knowledge);
}

@Override
protected void onPostExecute(String response) {
super.onPostExecute(response);
addMsg(response, 0);
if (audioBytes != null) {
int index = response.indexOf("|");
addMsg(response.substring(index+1), 1);
addMsg(response.substring(0, index), 0);
} else {
addMsg(response, 0);
}
}
}

Expand Down
66 changes: 48 additions & 18 deletions dialogflow/app/src/main/java/com/google/cloud/examples/dialogflow/utils/ApiRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import com.google.auth.Credentials;
import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.dialogflow.v2beta1.AudioEncoding;
import com.google.cloud.dialogflow.v2beta1.InputAudioConfig;
import com.google.cloud.dialogflow.v2beta1.DetectIntentRequest;
import com.google.cloud.dialogflow.v2beta1.DetectIntentResponse;
import com.google.cloud.dialogflow.v2beta1.KnowledgeAnswers;
Expand All @@ -35,6 +37,7 @@
import com.google.cloud.dialogflow.v2beta1.SessionsSettings;
import com.google.cloud.dialogflow.v2beta1.TextInput;
import com.google.cloud.examples.dialogflow.AppController;
import com.google.protobuf.ByteString;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
Expand Down Expand Up @@ -64,23 +67,26 @@ public ApiRequest() {
* @param knowledge : send message to knowledge base if true
* @return : response from the server
*/
public String callAPI(String accessToken, Date expiryTime, String msg, boolean tts,
public String callAPI(String accessToken, Date expiryTime, String msg, byte[] audioBytes, boolean tts,
boolean sentiment, boolean knowledge) {
this.token = accessToken;
this.tokenExpiration = expiryTime;
return detectIntent(msg, tts, sentiment, knowledge);

return detectIntent(msg, audioBytes, tts, sentiment, knowledge);
}

/**
* function to getting the results from the dialogflow
*
* @param msg : message sent by the user
* @param tts : send message to text to speech if true
* @param sentiment : send message to sentiment analysis if true
* @param knowledge : send message to knowledge base if true
* @param msg : message sent by the user
* @param audioBytes : audio sent by the user
* @param tts : send message to text to speech if true
* @param sentiment : send message to sentiment analysis if true
* @param knowledge : send message to knowledge base if true
* @return : response from the server
*/
private String detectIntent(String msg, boolean tts, boolean sentiment, boolean knowledge) {
private String detectIntent(String msg, byte[] audioBytes, boolean tts, boolean sentiment,
boolean knowledge) {
try {
AccessToken accessToken = new AccessToken(token, tokenExpiration);
Credentials credentials = GoogleCredentials.create(accessToken);
Expand All @@ -92,17 +98,30 @@ private String detectIntent(String msg, boolean tts, boolean sentiment, boolean
SessionName sessionName = SessionName.of(AppController.PROJECT_ID,
AppController.SESSION_ID);

// Set the text (hello) and language code (en-US) for the query
TextInput textInput = TextInput.newBuilder()
.setText(msg)
.setLanguageCode("en-US")
.build();

// Build the query with the TextInput
QueryInput queryInput = QueryInput.newBuilder().setText(textInput).build();
QueryInput queryInput;
if (msg != null) {
// Set the text (hello) and language code (en-US) for the query
TextInput textInput = TextInput.newBuilder()
.setText(msg)
.setLanguageCode("en-US")
.build();

// Build the query with the TextInput
queryInput = QueryInput.newBuilder().setText(textInput).build();
} else {
// Instructs the speech recognizer how to process the audio content.
InputAudioConfig inputAudioConfig = InputAudioConfig.newBuilder()
.setAudioEncoding(AudioEncoding.AUDIO_ENCODING_AMR)
.setLanguageCode("en-US")
.setSampleRateHertz(8000)
.build();

// Build the query with the TextInput
queryInput = QueryInput.newBuilder().setAudioConfig(inputAudioConfig).build();
}

DetectIntentRequest detectIntentRequest = getDetectIntentRequest(sessionName,
queryInput, tts, sentiment, knowledge, fixedCredentialsProvider);
queryInput, tts, sentiment, knowledge, fixedCredentialsProvider, audioBytes);

DetectIntentResponse detectIntentResponse =
sessionsClient.detectIntent(detectIntentRequest);
Expand All @@ -113,7 +132,14 @@ private String detectIntent(String msg, boolean tts, boolean sentiment, boolean
AppController.playAudio(detectIntentResponse.getOutputAudio().toByteArray());
}

return handleResults(detectIntentResponse);
if (msg != null) {
return handleResults(detectIntentResponse);
} else {
return String.format(
"%s|%s",
handleResults(detectIntentResponse),
detectIntentResponse.getQueryResult().getQueryText());
}
} catch (Exception ex) {
ex.printStackTrace();
return ex.getMessage();
Expand All @@ -133,12 +159,16 @@ private String detectIntent(String msg, boolean tts, boolean sentiment, boolean
*/
private DetectIntentRequest getDetectIntentRequest(SessionName sessionName,
QueryInput queryInput, boolean tts, boolean sentiment, boolean knowledge,
FixedCredentialsProvider fixedCredentialsProvider) throws Exception {
FixedCredentialsProvider fixedCredentialsProvider, byte[] audioBytes) throws Exception {

DetectIntentRequest.Builder detectIntentRequestBuilder = DetectIntentRequest.newBuilder()
.setSession(sessionName.toString())
.setQueryInput(queryInput);

if (audioBytes != null) {
detectIntentRequestBuilder.setInputAudio(ByteString.copyFrom(audioBytes));
}

QueryParameters.Builder queryParametersBuilder = QueryParameters.newBuilder();

if (tts) {
Expand Down
2 changes: 1 addition & 1 deletion dialogflow/app/src/main/res/values/colors.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
<resources>
<color name="colorPrimary">#ff9800</color>
<color name="colorPrimaryDark">#ef6c00</color>
<color name="colorAccent">#FF4081</color>
<color name="colorAccent">#ff9800</color>
<color name="colorHeading">#FFFFFF</color>
</resources>

0 comments on commit a49dd71

Please sign in to comment.