Skip to content

Commit

Permalink
[Modify] some fixes & optimize memory & optimize online (#26)
Browse files Browse the repository at this point in the history 
1. fix some errors in cyberdog_audio
2. optimize memory using in cyberdog_audio while recording and playing
3. update audio online work mode

Signed-off-by: northdk <[email protected]>
  • Loading branch information
@northdk
northdk authored and Homalozoa committed Oct 15, 2021
1 parent b35dc14 commit 132b607
Show file tree
Hide file tree
Showing 15 changed files with 579 additions and 315 deletions.
10 changes: 6 additions & 4 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@

Xiaomi CyberDog's first release.

## 1.0.1 (2021-09-16)
### 1.0.0.1 (2021-09-16)

Move the repositories to GitHub and replace the athena keyword with cyberdog in batches.

## 1.0.2 (2021-09-27)
### 1.0.0.2 (2021-09-27)

Add grpc_vendor, lcm_vendor, mpg123_vendor, sdl2main_vendor, sdl2mixer_vendor and opencv_vendor to constrain the building environment.

## 1.0.3 (2021-10-14)
### 1.0.0.3 (2021-10-14)

Remove opencv_vendor, combine sdl2main_vendor & sdl2mixer_vendor into sdl2_vendor. Remove vision pkgs.
- Remove opencv_vendor, combine sdl2main_vendor & sdl2mixer_vendor into sdl2_vendor. Remove vision pkgs.
- Add behaviortreecppv3_vendor
- Optimize audio pkgs
6 changes: 6 additions & 0 deletions cyberdog_interaction/cyberdog_audio/audio_assistant/include/audio_assistant/combiner.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@

#include <stdlib.h>
#include <cstdlib>
#include <memory>

#include "sys/stat.h"
#include "sys/types.h"

#include "audio_base/audio_player.hpp"
#include "audio_base/audio_queue.hpp"

#ifdef __cplusplus
extern "C"
Expand All @@ -30,6 +33,9 @@ extern "C"
#ifdef __cplusplus
}
#endif // __cplusplus

#define DELAY_TIMES 10

extern bool player_end;

void recorder_work_loop(void);
Expand Down
19 changes: 10 additions & 9 deletions cyberdog_interaction/cyberdog_audio/audio_assistant/src/ai_keyword.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "rclcpp/rclcpp.hpp"
#include "xiaoai_sdk/vpm/vpm_sdk.h"
#include "audio_base/debug/ai_debugger.hpp"
#include "audio_base/audio_queue.hpp"
#include "audio_assistant/mp3decoder.hpp"

#define VPM_CONFIG_FILE_PATH "/opt/ros2/cyberdog/ai_conf/xaudio_engine.conf"
Expand All @@ -36,10 +37,9 @@ typedef struct
{
unsigned char * start;
unsigned int length;
std::mutex mutex;
} vpm_input_buf_t;

extern vpm_input_buf_t vpm_input_buffer;
extern athena_audio::MsgQueue<vpm_input_buf_t> vpm_msg_queue;
extern bool vpm_buf_ready;
extern void ai_nativeasr_data_handler(asr_msg_type_t msg, unsigned char * buf, int buf_size);
void ai_vpm_work_loop();
Expand Down Expand Up @@ -154,9 +154,9 @@ static void vpm_wakeup_data_handle(
return;
}

// close interrupt while audio playing
// std::shared_ptr<mp3decoder> mp3decoder_ptr = mp3decoder::GetInstance();
// if ( mp3decoder_ptr->tts_player_is_playing() == true )
// {
// if ( mp3decoder_ptr->tts_player_is_playing() == true ) {
// aivs_set_interrupt();
// tts_stream_flush();
// mp3decoder_ptr->tts_player_stop_now();
Expand Down Expand Up @@ -341,6 +341,7 @@ static int setParas(int type, int value)

void ai_vpm_work_loop()
{
fprintf(stdout, "vc: ai vp work loop on call\n");
for (;; ) {
if (getAiWakeupRunStatus() == false) {
printf("vc:vpm: getAiWakeupRunStatus shitch off\n");
Expand All @@ -352,12 +353,12 @@ void ai_vpm_work_loop()
if (vpm_buf_ready == false) {
continue;
}
// will check here for open/close input to wakeup sdk
vpm_input_buffer.mutex.lock();
input.raw = reinterpret_cast<void *>(vpm_input_buffer.start);
input.size = vpm_input_buffer.length;

vpm_input_buf_t vpm_data;
vpm_msg_queue.DeQueue(vpm_data);
input.raw = reinterpret_cast<void *>(vpm_data.start);
input.size = vpm_data.length;
vpm_process(&input);
vpm_input_buffer.mutex.unlock();
}

/*release vpm*/
Expand Down
127 changes: 60 additions & 67 deletions cyberdog_interaction/cyberdog_audio/audio_assistant/src/ai_online/AivsMain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
#include "xiaoai_sdk/aivs/ClientLoggerHooker.h"
#include "xiaoai_sdk/audio_config/config.h"
#include "audio_assistant/audio_assitant.hpp"
#include "audio_base/audio_queue.hpp"
#include "audio_assistant/mp3decoder.hpp"

#include "xiaoai_sdk/aivs/RobotController.h"
Expand Down Expand Up @@ -96,6 +97,13 @@ typedef struct tts_info
tts_type_t type;
unsigned char * address;
unsigned int length;
void free_mem()
{
if (address != nullptr) {
fprintf(stdout, "vc: free ttsinfo memory\n");
free(address);
}
}
} tts_info_t;

typedef struct tts_queue
Expand All @@ -105,6 +113,8 @@ typedef struct tts_queue
} tts_queue_t;
static tts_queue_t mMsgQueueTts;

static athena_audio::MsgQueue<tts_info_t> TTSMsgQueue;

bool getAivsTtsRunStatus(void)
{
return mAivsTtsRunStatus;
Expand Down Expand Up @@ -139,12 +149,7 @@ int tts_player_decode_paly(void);

int tts_stream_flush(void)
{
// will check here
std::cout << "vc: befor clear, queue.size " << mMsgQueueTts.queue.size() << std::endl;
mMsgQueueTts.mutex.lock(); // queue lock
mMsgQueueTts.queue.clear();
mMsgQueueTts.mutex.unlock(); // queue unlock
std::cout << "vc: befor clear, queue.size " << mMsgQueueTts.queue.size() << std::endl;
TTSMsgQueue.Clear();
return 0;
}

Expand All @@ -159,47 +164,38 @@ int aivs_set_interrupt(void)
return 0;
}

int ai_push_msg_playback(tts_info_t ttsInfo)
int ai_push_msg_playback(const tts_info_t & ttsInfo)
{
std::cout << "vc: enter ai_push_msg_playback()" << std::endl;
tts_info_t ttsInfoPlayback = {TTS_TYPE_INVALID, NULL, 0};
tts_info_t ttsInfoPlayStart = {TTS_TYPE_INVALID, NULL, 0};
tts_info_t ttsInfoPlayStop = {TTS_TYPE_INVALID, NULL, 0};
std::cout << "vc: enter ai_push_msg_playback(), type: " << ttsInfo.type << std::endl;
tts_info_t ttsInfoPlay = {ttsInfo.type, NULL, ttsInfo.length};

switch (ttsInfo.type) {
case TTS_TYPE_STREAM:
std::cout << "vc: TTS_TYPE_STREAM Case !!" << std::endl;

if (ttsInfo.address == NULL || ttsInfo.length == 0) {
std::cout << "vc: ai_push_msg_playback invalid paramter !!!" << std::endl;
std::cout << "vc: ai_push_msg_playback invalid paramter, stream addr is null !!!" <<
std::endl;
return -1;
} else {
mMsgQueueTts.mutex.lock(); // queue lock
ttsInfoPlayback.type = ttsInfo.type;
ttsInfoPlayback.length = ttsInfo.length;
ttsInfoPlayback.address = (unsigned char *)malloc(ttsInfo.length);
if (ttsInfoPlayback.address != NULL) {
memcpy(ttsInfoPlayback.address, ttsInfo.address, ttsInfoPlayback.length);
mMsgQueueTts.queue.push_front(ttsInfoPlayback);
ttsInfoPlay.address = (unsigned char *)malloc(ttsInfo.length);
if (ttsInfoPlay.address != NULL) {
memcpy(ttsInfoPlay.address, ttsInfo.address, ttsInfoPlay.length);
TTSMsgQueue.EnQueue(ttsInfoPlay);
} else {
std::cout << "vc: ai_push_msg_playback failed, cannot allocate memory!" << std::endl;
}
mMsgQueueTts.mutex.unlock(); // queue unlock
}
break;

case TTS_TYPE_SYNC_START:
std::cout << "vc: TTS_TYPE_SYNC_START Case !!" << std::endl;
mMsgQueueTts.mutex.lock(); // queue lock
ttsInfoPlayStart.type = ttsInfo.type;
mMsgQueueTts.queue.push_front(ttsInfoPlayStart);
mMsgQueueTts.mutex.unlock(); // queue unlock
TTSMsgQueue.EnQueue(ttsInfoPlay);
break;

case TTS_TYPE_SYNC_STOP:
std::cout << "vc: TTS_TYPE_SYNC_STOP Case !!" << std::endl;
mMsgQueueTts.mutex.lock(); // queue lock
ttsInfoPlayStop.type = ttsInfo.type;
mMsgQueueTts.queue.push_front(ttsInfoPlayStop);
mMsgQueueTts.mutex.unlock(); // queue unlock
TTSMsgQueue.EnQueue(ttsInfoPlay);
break;

default:
Expand All @@ -222,51 +218,46 @@ int aivsTtsHandler(void)
continue;
}

mMsgQueueTts.mutex.lock(); // queue lock
if (!mMsgQueueTts.queue.empty()) {
ttsInfoPlayback = mMsgQueueTts.queue.back();
mMsgQueueTts.queue.pop_back();
mMsgQueueTts.mutex.unlock(); // queue unlock
switch (ttsInfoPlayback.type) {
case TTS_TYPE_STREAM:
if (ttsInfoPlayback.address == NULL || ttsInfoPlayback.length == 0) {
usleep(1000 * 100);
continue;
} else {
std::cout << "vc: Process TTS_TYPE_STREAM " << std::endl;
cyberdog_audio::mp3decoder::GetInstance()->tts_player_accumulate(
ttsInfoPlayback.address,
ttsInfoPlayback.length);
free(ttsInfoPlayback.address);
}
break;
if (!TTSMsgQueue.DeQueue(ttsInfoPlayback)) {
std::cout << "vc: aivsTtsHandler get msg with empty, skip once!" << std::endl;
continue;
}

case TTS_TYPE_SYNC_START:
std::cout << "vc: Process TTS_TYPE_SYNC_START " << std::endl;
cyberdog_audio::mp3decoder::GetInstance()->tts_player_init();
ai_push_msg(101); /*led control for start-dialog status*/
break;
switch (ttsInfoPlayback.type) {
case TTS_TYPE_STREAM:
if (ttsInfoPlayback.address == NULL || ttsInfoPlayback.length == 0) {
usleep(1000 * 100);
continue;
} else {
std::cout << "vc: Process TTS_TYPE_STREAM " << std::endl;
cyberdog_audio::mp3decoder::GetInstance()->tts_player_accumulate(
ttsInfoPlayback.address,
ttsInfoPlayback.length);
free(ttsInfoPlayback.address);
}
break;

case TTS_TYPE_SYNC_STOP:
std::cout << "vc: Process TTS_TYPE_SYNC_STOP " << std::endl;
case TTS_TYPE_SYNC_START:
std::cout << "vc: Process TTS_TYPE_SYNC_START " << std::endl;
cyberdog_audio::mp3decoder::GetInstance()->tts_player_init();
ai_push_msg(101); /*led control for start-dialog status*/
break;

if (cyberdog_audio::mp3decoder::GetInstance() == nullptr) {
fprintf(stderr, "Process TTS_TYPE_SYNC_STOP Error with invalid pointer!\n");
break;
}
case TTS_TYPE_SYNC_STOP:
std::cout << "vc: Process TTS_TYPE_SYNC_STOP " << std::endl;

cyberdog_audio::mp3decoder::GetInstance()->tts_player_decode_paly();
// ai_push_msg(104);/*led control for end -dialog status*/
if (cyberdog_audio::mp3decoder::GetInstance() == nullptr) {
fprintf(stderr, "Process TTS_TYPE_SYNC_STOP Error with invalid pointer!\n");
break;
}

default:
std::cout << "vc: Process default case !!" << std::endl;
break;
}
} else {
mMsgQueueTts.mutex.unlock(); // queue unlock
usleep(1000 * 100);
continue;
cyberdog_audio::mp3decoder::GetInstance()->tts_player_decode_paly();
// ai_push_msg(104);/*led control for end -dialog status*/
break;

default:
std::cout << "vc: Process default case !!" << std::endl;
break;
}
}
}
Expand Down Expand Up @@ -778,6 +769,8 @@ void initAivsDeviceOAuth()
clientInfo->setDeviceId(DEVICE_OAUTH_DEVICE_ID);

auto config = getAudioConfig();
config->putBoolean(aivs::AivsConfig::Auth::REQ_TOKEN_HYBRID, true);
config->putBoolean(aivs::AivsConfig::Auth::REQ_TOKEN_BY_SDK, true);
gEngine = aivs::Engine::create(config, clientInfo, aivs::Engine::ENGINE_AUTH_DEVICE_OAUTH);
}

Expand Down
5 changes: 3 additions & 2 deletions cyberdog_interaction/cyberdog_audio/audio_assistant/src/ai_online/StorageCapabilityImpl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@

StorageCapabilityImpl::StorageCapabilityImpl()
{
#ifndef __NuttX__
/* #ifndef __NuttX__
storageFilePath = "data.json";
#else
storageFilePath = "/tmp/data.json";
#endif
#endif */
storageFilePath = "/opt/ros2/cyberdog/data/token.json";
loadKeyValuesFromFile();
}

Expand Down
Loading

0 comments on commit 132b607

Please sign in to comment.