/** * Copyright (2017) Baidu Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * File: duerapp_recorder.c * Auth: Renhe Zhang (v_zhangrenhe@baidu.com) * Desc: Record module function implementation. */ #include #include #include #include #include #ifdef ENABLE_TCP_NODELAY #include #endif #include #include #include #include #include #include #include #include "duerapp_recorder.h" #include "duerapp_config.h" #include "lightduer_voice.h" #include "lightduer_dcs_router.h" #include #include "snowboy-detect-c-wrapper.h" #define ALSA_PCM_NEW_HW_PARAMS_API #define SAMPLE_RATE (16000) #define FRAMES_INIT (640*4) #define CHANNEL (2) #define FRAMES_SIZE ((16/8) *CHANNEL)// bytes / sample * channels //#define PCM_STREAM_CAPTURE_DEVICE "hw:1,0" #define PCM_STREAM_CAPTURE_DEVICE "default" //#define RECORD_DATA_TO_FILE static int s_duer_rec_snd_fd=-1; static int s_duer_rec_recv_fd=-1; struct sockaddr_in s_duer_rec_addr; static duer_rec_state_t s_duer_rec_state = RECORDER_STOP; static pthread_t s_rec_threadID; static sem_t s_rec_sem; static duer_rec_config_t *s_index = NULL; static bool s_is_suspend = false; static bool s_is_baidu_rec_start = false; static pthread_t s_rec_send_threadID; static char * s_kws_model_filename = NULL; const char *s_tone_url[3] = {"./resources/60.wav","./resources/61.wav","./resources/62.wav"}; extern void event_record_start(); int duer_set_kws_model_file(char *filename) { if(filename==NULL){ return 0; } if(s_kws_model_filename==NULL){ free(s_kws_model_filename); s_kws_model_filename=NULL; } s_kws_model_filename = strdup(filename); return 0; } typedef struct _pcm_header_t { char pcm_header[4]; size_t pcm_length; char format[8]; int bit_rate; short pcm; short channel; int sample_rate; int byte_rate; short block_align; short bits_per_sample; char fix_data[4]; size_t data_length; } pcm_header_t; int g_recorder_channel = 0; static pcm_header_t s_pcm_header = { {'R', 'I', 'F', 'F'}, (size_t)-1, {'W', 'A', 'V', 'E', 'f', 'm', 't', ' '}, 0x10, 0x01, 0x01, #if 1 //def SAMPLE_RATE_16K 0x3E80, 0x7D00, #else //8k 0x1F40, 0x3E80, #endif // SAMPLE_RATE_16K 0x02, 0x10, {'d', 'a', 't', 'a'}, (size_t)-1 }; static FILE *s_voice_file=NULL; int duer_store_voice_start(int name_id) { DUER_LOGD("start"); if (s_voice_file) { fclose(s_voice_file); s_voice_file = NULL; } char _name[64]; snprintf(_name, sizeof(_name), "./%d.wav", name_id); s_voice_file = fopen(_name, "wb"); if (!s_voice_file) { DUER_LOGE("can't open file %s", _name); return -1; } else { DUER_LOGD("begin write to file:%s", _name); } fwrite(&s_pcm_header, 1, sizeof(s_pcm_header), s_voice_file); s_pcm_header.data_length = 0; s_pcm_header.pcm_length = sizeof(s_pcm_header) - 8; return 0; } int duer_store_voice_write(const void *data, uint32_t size) { DUER_LOGD("data"); if (s_voice_file) { fwrite(data, 1, size, s_voice_file); s_pcm_header.data_length += size; } return 0; } int duer_store_voice_end() { if (s_voice_file) { s_pcm_header.pcm_length += s_pcm_header.data_length; fseek(s_voice_file, 0, SEEK_SET); fwrite(&s_pcm_header, 1, sizeof(s_pcm_header), s_voice_file); fclose(s_voice_file); s_voice_file = NULL; DUER_LOGD("finish the voice record"); } return 0; } int stereo_to_mono(int16_t *in,int ilen,int16_t *out,int outlen) { int i=0; if(CHANNEL==1){ memcpy(out,in,ilen<<1); return ilen; }else if(CHANNEL==2){ for(i=0;i>1;i++){ if(duer_app_is_test_mode()!=1||(g_recorder_channel==0)){ out[i] = (in[2*i]+in[2*i+1])>>1; //out[i] = in[2*i]; }else if(g_recorder_channel==1){ out[i] = in[2*i]; }else if(g_recorder_channel==2){ out[i] = in[2*i+1]; } } return ilen>>1; }else if(CHANNEL==4){ for(i=0;i>2;i++){ out[i] = (in[4*i]+in[4*i+1]+in[4*i+2]+in[4*i+3])>>2; //out[i] = in[4*i+2]; } return ilen>>2; } return 0; } int duer_recorder_test_start(int channel) { g_recorder_channel = channel; if(duer_app_is_test_mode()){ if(g_recorder_channel==1){ duer_media_tone_play("./resources/left.wav",20000); }else if(g_recorder_channel==2){ duer_media_tone_play("./resources/right.wav",20000); } duer_store_voice_end(); duer_store_voice_start(g_recorder_channel); } return 0; } int duer_recorder_test_end(void) { char fn[32]; sprintf(fn,"./%d.wav",g_recorder_channel); duer_store_voice_end(); if(duer_app_is_test_mode()){ duer_media_tone_play(fn,20000); } return 0; } static void recorder_thread() { fd_set fdwrite; int value=0; const char resource_filename[] = "resources/common.res"; //const char model_filename[] = "snowboy/resources/models/snowboy.umdl"; //const char model_filename[] = "resources/models/keywords.pmdl,resources/models/peppapig.pmdl"; //const char sensitivity_str[] = "0.5,0.5"; //const char model_filename[] = "resources/models/keywords.pmdl"; char *model_filename=s_kws_model_filename; //const char sensitivity_str[] = "0.5,0.6,0.6"; const char sensitivity_str[] = "0.5"; float audio_gain = 1.1; bool apply_frontend = false; if(model_filename==NULL){ model_filename = "resources/models/keywords.pmdl"; } // Initializes Snowboy detector. SnowboyDetect* detector = SnowboyDetectConstructor(resource_filename, (const char*)model_filename); SnowboyDetectSetSensitivity(detector, sensitivity_str); SnowboyDetectSetAudioGain(detector, audio_gain); SnowboyDetectApplyFrontend(detector, apply_frontend); // Initializes PortAudio. You may use other tools to capture the audio. value = SnowboyDetectSampleRate(detector); DUER_LOGI("samplerate:%d\n",value); value = SnowboyDetectNumChannels(detector); DUER_LOGI("channels:%d\n",value); value = SnowboyDetectBitsPerSample(detector); DUER_LOGI("bits:%d\n",value); pthread_detach(pthread_self()); snd_pcm_hw_params_get_period_size(s_index->params, &(s_index->frames), &(s_index->dir)); if (s_index->frames < 0) { DUER_LOGE("Get period size failed!"); return; } DUER_LOGI("frames %d dir %d\n",s_index->frames,s_index->dir); s_index->size = s_index->frames * FRAMES_SIZE; int16_t *buffer = NULL; int16_t *mono_buffer = NULL; int mono_data_size = 0; if (buffer) { free(buffer); buffer = NULL; } buffer = (int16_t *)malloc(s_index->size); if (!buffer) { DUER_LOGE("malloc buffer failed!\n"); } else { memset(buffer, 0, s_index->size); } mono_buffer = (int16_t *)malloc(s_index->size); if (!mono_buffer) { DUER_LOGE("malloc buffer failed!\n"); } else { memset(mono_buffer, 0, s_index->size); } while (1) { int ret = snd_pcm_readi(s_index->handle, buffer, s_index->frames); if (ret == -EPIPE) { DUER_LOGE("an overrun occurred!"); snd_pcm_prepare(s_index->handle); continue; } else if (ret < 0) { DUER_LOGE("read: %s", snd_strerror(ret)); continue; } else if (ret != (int)s_index->frames) { DUER_LOGE("read %d frames!", ret); continue; } else { // do nothing //printf("ret=%d\n",ret); } mono_data_size = stereo_to_mono(buffer,s_index->size>>1,mono_buffer,s_index->size>>1); #if 1 int result = SnowboyDetectRunDetection(detector, mono_buffer, mono_data_size, false); if (result > 0) { DUER_LOGI("Hotword %d detected!\n", result); duer_dcs_dialog_cancel(); duer_media_tone_play(s_tone_url[rand()%3],5000); event_record_start(); #ifdef RECORD_DATA_TO_FILE duer_store_voice_end(); duer_store_voice_start(time(NULL)); #else if(duer_app_is_test_mode()){ duer_store_voice_end(); duer_store_voice_start(g_recorder_channel); } #endif } #endif if((RECORDER_START == s_duer_rec_state)&&s_is_baidu_rec_start){ struct timeval time = {0, 0}; FD_ZERO(&fdwrite); FD_SET(s_duer_rec_snd_fd, &fdwrite); if(select(s_duer_rec_snd_fd + 1, NULL, &fdwrite, NULL, &time)>0){ send(s_duer_rec_snd_fd,mono_buffer,mono_data_size<<1,0); #ifdef RECORD_DATA_TO_FILE duer_store_voice_write(mono_buffer,mono_data_size<<1); #else if(duer_app_is_test_mode()){ DUER_LOGI("test mode\n"); duer_store_voice_write(mono_buffer,mono_data_size<<1); } #endif }else{ DUER_LOGI("overloap!!!!!!!!!\n"); } } } if (buffer) { free(buffer); buffer = NULL; } if(mono_buffer){ free(mono_buffer); mono_buffer=NULL; } snd_pcm_drain(s_index->handle); snd_pcm_close(s_index->handle); if(s_index) { free(s_index); s_index = NULL; } SnowboyDetectDestructor(detector); return; } static void recorder_data_send_thread() { char *buffer = NULL; int size = s_index->size; fd_set fdread; struct timeval time = {0, 0}; int recvlen=0; pthread_detach(pthread_self()); DUER_LOGI("recorder_data_send_thread start!\n"); s_is_baidu_rec_start = false; while(1) { FD_ZERO(&fdread); FD_SET(s_duer_rec_recv_fd, &fdread); if(select(s_duer_rec_recv_fd + 1, &fdread, NULL, NULL, &time)>0){ recv(s_duer_rec_recv_fd, buffer, size, 0); }else{ break; } } DUER_LOGI("flush data end %d!\n",s_duer_rec_state); s_is_baidu_rec_start = true; duer_voice_start(16000); buffer = (char *)malloc(size); if (!buffer) { DUER_LOGE("malloc buffer failed!\n"); } else { memset(buffer, 0, size); } while (RECORDER_START == s_duer_rec_state) { FD_ZERO(&fdread); time.tv_sec = 1; time.tv_usec = 0; FD_SET(s_duer_rec_recv_fd, &fdread); if(select(s_duer_rec_recv_fd + 1, &fdread, NULL, NULL, &time)>0){ recvlen = recv(s_duer_rec_recv_fd, buffer, size, 0); if(recvlen>0){ printf(".&."); duer_voice_send(buffer, recvlen); }else{ DUER_LOGE("recv fail!\n"); } } } s_is_baidu_rec_start = false; duer_voice_stop(); if(s_is_suspend) { duer_voice_terminate(); s_is_suspend = false; } if (buffer) { free(buffer); buffer = NULL; } if(sem_post(&s_rec_sem)) { DUER_LOGE("sem_post failed."); } return ; } static int duer_open_alsa_pcm() { int ret = DUER_OK; int result = (snd_pcm_open(&(s_index->handle), PCM_STREAM_CAPTURE_DEVICE, SND_PCM_STREAM_CAPTURE, 0)); if (result < 0) { DUER_LOGE("\n\n****unable to open pcm device: %s*********\n\n", snd_strerror(ret)); ret = DUER_ERR_FAILED; } return ret; } static int duer_set_pcm_params() { int ret = DUER_OK; snd_pcm_hw_params_alloca(&(s_index->params)); snd_pcm_hw_params_any(s_index->handle, s_index->params); snd_pcm_hw_params_set_access(s_index->handle, s_index->params, SND_PCM_ACCESS_RW_INTERLEAVED); snd_pcm_hw_params_set_format(s_index->handle, s_index->params, SND_PCM_FORMAT_S16_LE); snd_pcm_hw_params_set_channels(s_index->handle, s_index->params, CHANNEL); snd_pcm_hw_params_set_rate_near(s_index->handle, s_index->params, &(s_index->val), &(s_index->dir)); snd_pcm_hw_params_set_period_size_near(s_index->handle, s_index->params, &(s_index->frames), &(s_index->dir)); int result = snd_pcm_hw_params(s_index->handle, s_index->params); if (result < 0) { DUER_LOGE("unable to set hw parameters: %s", snd_strerror(ret)); ret = DUER_ERR_FAILED; } return ret; } int duer_recorder_start() { int ret = 0; DUER_LOGI("duer_recorder_start %d!",s_duer_rec_state); do { if(sem_wait(&s_rec_sem)) { DUER_LOGE("sem_wait failed."); break; } if (RECORDER_STOP == s_duer_rec_state) { s_duer_rec_state = RECORDER_START; ret = pthread_create(&s_rec_send_threadID, NULL, (void *)recorder_data_send_thread, NULL); if(ret != 0) { s_duer_rec_state = RECORDER_STOP; DUER_LOGE("Create recorder pthread error!"); break; } else { pthread_setname_np(s_rec_send_threadID, "recorder"); } } else{ DUER_LOGI("Recorder Start failed! state:%d", s_duer_rec_state); } return DUER_OK; } while(0); return DUER_ERR_FAILED; } int duer_recorder_stop() { int ret = DUER_OK; DUER_LOGI("duer_recorder_stop! state:%d", s_duer_rec_state); if (RECORDER_START == s_duer_rec_state) { s_duer_rec_state = RECORDER_STOP; s_is_baidu_rec_start = false; } else { ret = DUER_ERR_FAILED; DUER_LOGI("Recorder Stop failed! state:%d", s_duer_rec_state); } return ret; } int duer_recorder_suspend() { int ret = duer_recorder_stop(); if (DUER_OK == ret) { s_is_suspend = true; } else { DUER_LOGI("Recorder Stop failed! state:%d", s_duer_rec_state); } return ret; } duer_rec_state_t duer_get_recorder_state() { return s_duer_rec_state; } int duer_hotwords_detect_start(char *model_filename) { int ret=0; duer_set_kws_model_file(model_filename); s_duer_rec_addr.sin_family = AF_INET; s_duer_rec_addr.sin_port = htons(19290); s_duer_rec_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); if(sem_init(&s_rec_sem, 0, 1)) { DUER_LOGE("Init s_rec_sem failed."); return -1; } s_index = (duer_rec_config_t *)malloc(sizeof(duer_rec_config_t)); if (!s_index) { DUER_LOGE("malloc fail"); return -1; } memset(s_index, 0, sizeof(duer_rec_config_t)); s_index->frames = FRAMES_INIT; s_index->val = SAMPLE_RATE; // pcm sample rate do{ s_duer_rec_snd_fd = socket(AF_INET, SOCK_DGRAM,IPPROTO_UDP); if(s_duer_rec_snd_fd <0){ DUER_LOGE("open send socket failed"); break; } ret = connect(s_duer_rec_snd_fd, (struct sockaddr *)&s_duer_rec_addr, sizeof(s_duer_rec_addr)); if(ret!=0){ DUER_LOGE("socket connect failed"); break; } s_duer_rec_recv_fd = socket(AF_INET, SOCK_DGRAM,IPPROTO_UDP); if(s_duer_rec_recv_fd <0){ DUER_LOGE("open recv socket failed"); break; } ret = bind(s_duer_rec_recv_fd, (struct sockaddr *) &s_duer_rec_addr, sizeof(s_duer_rec_addr)); if(ret<0){ DUER_LOGE("socket bind failed"); break; } ret = duer_open_alsa_pcm(); if (ret != DUER_OK) { DUER_LOGE("open pcm failed"); break; } ret = duer_set_pcm_params(); if (ret != DUER_OK) { DUER_LOGE("open pcm failed"); break; } ret = pthread_create(&s_rec_threadID, NULL, (void *)recorder_thread, NULL); if(ret != 0){ DUER_LOGE("Create recorder pthread error!"); break; } else { pthread_setname_np(s_rec_threadID, "recorder"); } }while(0); if(ret!=0){ if(s_index) { free(s_index); s_index = NULL; } if(s_duer_rec_recv_fd>=0){ close(s_duer_rec_recv_fd); s_duer_rec_recv_fd = -1; } if(s_duer_rec_snd_fd>=0){ close(s_duer_rec_snd_fd); s_duer_rec_snd_fd = -1; } } return ret; }