#include "asterisk.h" ASTERISK_FILE_VERSION(__FILE__, "$Revision: $") /* Asterisk-Sphinx Generic Speech API Module * (c) 2009, Christopher Jansen * * Distributed under whatever is the freest license I'm allowed * under Asterisk, I'd assume that's the GPL. */ #include #include #include #include #include #include #include #include #include #include #include "asterisk/module.h" #include "asterisk/logger.h" #include "asterisk/strings.h" #include "asterisk/config.h" #include "asterisk/frame.h" #include "asterisk/dsp.h" #include "asterisk/speech.h" #include "asterisk/speech_sphinx.h" #define SPHINX_BUFSIZE 2048 #define SPHINX_ERROR 0 #define SPHINX_SUCCESS 1 /* Functions used internally only */ /* logs the current state as a NOTICE */ void log_state(struct ast_speech *speech); /* Establish socket connection to sphinx server */ int sphinx_connect(struct ast_speech *speech, char const *host, const int port); /* Disconnect socket */ int sphinx_disconnect(struct ast_speech *speech); /* exchange packets with server */ int sphinx_comm(struct sphinx_request *sr, struct ast_speech *speech, int catchup); /* clear all data */ int reinit_speech_data(struct ast_speech *speech); /* destroy all data */ int destroy_speech_data(struct ast_speech *speech); int sphinx_set_blocking(int s, int shouldblock); int sphinx_swrite(struct sphinx_state *ss, void *data, int len); int sphinx_sread(struct sphinx_state *ss, struct ast_speech *speech); int make_error(struct ast_speech *speech, char *errmsg); static struct ast_speech_engine SPHINX_ENGINE_INFO = { "Sphinx", sphinx_create, sphinx_destroy, sphinx_load, sphinx_unload, sphinx_activate, sphinx_deactivate, sphinx_write, sphinx_dtmf, sphinx_start, sphinx_change, sphinx_change_results_type, sphinx_get, AST_FORMAT_SLINEAR }; char SPHINX_SERVER_ADDR[] = "127.000.000.001"; int SPHINX_SERVER_PORT = 0; int SPHINX_SILENCE_TIME = 200; int SPHINX_NOISE_FRAMES = 0; int SPHINX_SILENCE_THRESHOLD = 500; int sphinx_set_blocking(int s, int shouldblock) { int block = shouldblock ? 0 : O_NONBLOCK; int sflags; if((sflags = fcntl(s, F_GETFL, 0)) == -1) return SPHINX_ERROR; if (fcntl(s, F_SETFL, sflags | block) == -1) return SPHINX_ERROR; return SPHINX_SUCCESS; } static int load_module(void) { if(ast_speech_register(&SPHINX_ENGINE_INFO)) { ast_log(LOG_ERROR, "Failed to register.\n"); return AST_MODULE_LOAD_FAILURE; } struct ast_config *conf = ast_config_load("sphinx.conf"); if(conf == NULL) { ast_log(LOG_ERROR, "Unable to load sphinx.conf\n"); return AST_MODULE_LOAD_FAILURE; } char const *value; if((value = ast_variable_retrieve(conf, "general", "serverip"))) { ast_copy_string(SPHINX_SERVER_ADDR, value, sizeof(SPHINX_SERVER_ADDR)); } if((value = ast_variable_retrieve(conf, "general", "serverport"))) { sscanf(value,"%d",&SPHINX_SERVER_PORT); } if((value = ast_variable_retrieve(conf, "general", "silencetime"))) { sscanf(value,"%d",&SPHINX_SILENCE_TIME); } if((value = ast_variable_retrieve(conf, "general", "noiseframes"))) { sscanf(value,"%d",&SPHINX_NOISE_FRAMES); } if((value = ast_variable_retrieve(conf, "general", "silencethreshold"))) { sscanf(value,"%d",&SPHINX_SILENCE_THRESHOLD); } ast_log(LOG_NOTICE, "Using Server: %s:%d Silence Time: %d Threshold: %d Noise Frames: %d\n", SPHINX_SERVER_ADDR, SPHINX_SERVER_PORT, SPHINX_SILENCE_TIME, SPHINX_SILENCE_THRESHOLD, SPHINX_NOISE_FRAMES); return AST_MODULE_LOAD_SUCCESS; } static int unload_module(void) { if(ast_speech_unregister(SPHINX_ENGINE_INFO.name)) { ast_log(LOG_ERROR, "Failed to unregister.\n"); return -1; } return 0; } // Needs to check status of connection; shoudl return non-zero to signal error int sphinx_create(struct ast_speech *speech) { ast_log(LOG_DEBUG,"sphinx_create called\n"); if(reinit_speech_data(speech) == SPHINX_SUCCESS) if(sphinx_connect(speech, SPHINX_SERVER_ADDR, SPHINX_SERVER_PORT) == SPHINX_SUCCESS) return 0; ast_log(LOG_ERROR, "Can't create Sphinx server\n"); return -1; } int sphinx_destroy(struct ast_speech *speech) { ast_log(LOG_DEBUG,"sphinx_destroy called\n"); if(sphinx_disconnect(speech) == SPHINX_SUCCESS) if(destroy_speech_data(speech) == SPHINX_SUCCESS) return SPHINX_SUCCESS; return SPHINX_ERROR; } // load and unload grammar will probably remain unimplemented; I cannot // see a reason to need this now. int sphinx_load(struct ast_speech *speech, char *grammar_name, char *grammar) { ast_log(LOG_DEBUG,"sphinx_load called with request for grammar %s\n", grammar_name); return 0; } int sphinx_unload(struct ast_speech *speech, char *grammar_name) { ast_log(LOG_DEBUG,"sphinx_unload called for grammar %s\n", grammar_name); return 0; } // Activte implemented and works. int sphinx_activate(struct ast_speech *speech, char *grammar_name) { ast_log(LOG_DEBUG,"sphinx_activate called for grammar %s\n", grammar_name); struct sphinx_request sr; sr.rtype = REQTYPE_GRAMMAR; sr.dlen = strlen(grammar_name) + 1; sr.data = grammar_name; if(sphinx_comm(&sr, speech, 1) != SPHINX_SUCCESS) { ast_log(LOG_ERROR, "Comms error changing grammar request\n"); ast_speech_change_state(speech,AST_SPEECH_STATE_NOT_READY); return -1; } ast_speech_change_state(speech,AST_SPEECH_STATE_READY); return 0; } // Deactivate does nothing to the grammar but based on the examples // I've seen it's a good place to signal to the engine this is its // last chance to provide results. int sphinx_deactivate(struct ast_speech *speech, char *grammar_name) { // ast_log(LOG_NOTICE,"sphinx_deactivate called for grammar name %s\n", grammar_name); struct sphinx_state * ss=(struct sphinx_state *)speech->data; if(ss == NULL) { ast_log(LOG_ERROR,"Error, no state\n"); return -1; } if(speech->state != AST_SPEECH_STATE_DONE && !ss->final) { // ast_log(LOG_NOTICE, "Not done so getting final results now...\n"); if(sphinx_write(speech, NULL, 0) != 0) { ast_log(LOG_ERROR,"Comms error - setting NOT_READY\n"); ast_speech_change_state(speech,AST_SPEECH_STATE_NOT_READY); return -1; } } return 0; } int sphinx_sread(struct sphinx_state *ss, struct ast_speech *speech) { if(ss->preads == 0 && ss->prbytes == 0) { // Nothing pending to read. // ast_log(LOG_NOTICE, "Sphinx sread called, no pending reads."); return SPHINX_SUCCESS; } int rbytes = 0; while(ss->preads && !ss->prbytes) { int32_t rsize = 0; if((rbytes = read(ss->s, &rsize, sizeof(int32_t))) == -1) { if(errno != EWOULDBLOCK) { return make_error(speech, strerror(errno)); } else { //ast_log(LOG_NOTICE, "Want to read rsize (%d pending) but would block\n", ss->preads); break; } } else if(rbytes == sizeof(int32_t)) { ss->rbufused = 0; ss->prbytes = rsize; ss->preads--; //ast_log(LOG_NOTICE, "Got rsize: %d, pending reads: %d\n", ss->prbytes, ss->preads); } } if(ss->prbytes + ss->rbufused > SPHINX_BUFSIZE) return make_error(speech, "BUFFER OVERFLOW IN SPHINX READ BUFFER\n"); rbytes = 0; while(ss->prbytes != 0 && (rbytes = read(ss->s, ss->rbuf + ss->rbufused, ss->prbytes)) != -1) { if(rbytes != 0) { ss->prbytes -= rbytes; ss->rbufused += rbytes; //ast_log(LOG_NOTICE, "Read %d bytes data, %d pending\n", rbytes, ss->prbytes); } } if(rbytes == -1) { if(errno == EWOULDBLOCK) { rbytes = 0; } else { return make_error(speech, strerror(errno)); } } if(rbytes && !ss->prbytes) // We finished reading some results. { if(speech->results == NULL) speech->results = ast_calloc(sizeof(struct ast_speech_result),1); if(speech->results == NULL) return make_error(speech, "Cannot allocate results\n"); int32_t new_score = *(int32_t*)ss->rbuf; if(new_score >= speech->results->score) { speech->results->score = new_score; if(speech->results->text != NULL) { free(speech->results->text); speech->results->text = NULL; } speech->results->text = ast_strndup(ss->rbuf + sizeof(int32_t), ss->rbufused - sizeof(int32_t)); ast_log(LOG_NOTICE, "Score: %d Result: '%s'\n", speech->results->score, speech->results->text); } else { ast_log(LOG_NOTICE, "New result with lower score; ignoring.\n"); } speech->flags |= AST_SPEECH_HAVE_RESULTS; } return SPHINX_SUCCESS; } int make_error(struct ast_speech *speech, char *errmsg) { ast_log(LOG_ERROR, "%s", errmsg); ast_speech_change_state(speech,AST_SPEECH_STATE_NOT_READY); return SPHINX_ERROR; } int sphinx_swrite(struct sphinx_state *ss, void *indata, int len) { char *data = (char *)indata; if(ss->pwbytes + len > SPHINX_BUFSIZE) // Too much data! { ast_log(LOG_ERROR, "Output buffer overflow.\n"); return SPHINX_ERROR; } memcpy(ss->sbuf + ss->pwbytes, data, len); ss->pwbytes += len; if(ss->pwbytes) // Something to send { int bcount = write(ss->s, ss->sbuf, ss->pwbytes); if(bcount == -1 && (errno != EWOULDBLOCK)) { ast_log(LOG_ERROR, "Error writing to Sphinx server: %s\n", strerror(errno)); return SPHINX_ERROR; } if(bcount == -1) bcount = 0; memmove(ss->sbuf, ss->sbuf + bcount, ss->pwbytes - bcount); ss->pwbytes -= bcount; // ast_log(LOG_NOTICE, "Wrote %d bytes, %d pending\n", bcount, ss->pwbytes); } return SPHINX_SUCCESS; } /* Does the 'big job' of getting data to/from the Sphinx Server. The comm protocol * if pretty stupid simple; we send a two-int header consisting of the length * of data to follow, then the type of request we are sending, then the data. We * expect a similar response, only without the type of request. */ int sphinx_comm(struct sphinx_request *sr, struct ast_speech *speech, int catchup) { if(speech == NULL) return make_error(speech, "No data\n"); struct sphinx_state * ss=(struct sphinx_state *)speech->data; if(ss == NULL) return make_error(speech,"No state\n"); if(ss->s == 0) return make_error(speech,"No socket\n"); if(sr == NULL) return make_error(speech,"No request\n"); if((sr->rtype & (REQTYPE_FINISH | REQTYPE_DATA)) && (speech->state == AST_SPEECH_STATE_DONE || ss->final)) { // ast_log(LOG_NOTICE, "Final results are in so not communicating data.\n"); sr->dlen = 0; } else { // Write request type, data length if(sphinx_swrite(ss, &sr->dlen, sizeof(sr->dlen)) != SPHINX_SUCCESS) return make_error(speech, "Socket write error sending dlen\n"); if(sphinx_swrite(ss, &sr->rtype, sizeof(sr->rtype)) != SPHINX_SUCCESS) return make_error(speech, "Socket write error sending rtype\n"); // Write actual data, if any if(sr->dlen) { if(sphinx_swrite(ss, sr->data, sr->dlen) != SPHINX_SUCCESS) return make_error(speech, "Socket write error sending data\n"); } ss->preads++; // Each request makes us expect a response. //ast_log(LOG_NOTICE, "Added pending read, now %d\n", ss->preads); if((sr->rtype == REQTYPE_DATA && sr->dlen == 0) || sr->rtype == REQTYPE_FINISH) // didn't send anything; we are done { // ast_log(LOG_NOTICE, "Final results, setting status to DONE.\n"); ast_speech_change_state(speech, AST_SPEECH_STATE_DONE); ss->final = 1; // log_state(speech); } } // ok, so we need a chance to read some data, and here it is. Normally, we just want to call read and it'll do what // it can, but if we are final, then we gotta make sure it finishes up. if(sphinx_sread(ss, speech) != SPHINX_SUCCESS) return SPHINX_ERROR; if(speech->state == AST_SPEECH_STATE_DONE || ss->final || catchup) { // ast_log(LOG_NOTICE, "Finishing pending reads: %d, bytes in current read: %d\n", ss->preads, ss->prbytes); while(ss->preads || ss->prbytes || ss->pwbytes) { //ast_log(LOG_NOTICE, "Flushing buffers, Responses Pending: %d, Bytes in current response: %d, Bytes to write: %d\n", // ss->preads, ss->prbytes, ss->pwbytes); fd_set rsel, wsel; struct timeval tv; int selret; FD_ZERO(&rsel); FD_ZERO(&wsel); if(ss->pwbytes) FD_SET(ss->s, &wsel); if(ss->prbytes || ss->preads) FD_SET(ss->s, &rsel); // 5 seconds timeout is extreme. tv.tv_sec = 5; tv.tv_usec = 0; selret = select(ss->s + 1, &rsel, &wsel, NULL, &tv); if(selret == -1) return make_error(speech, "Select returned error.\n"); else if(selret == 0) { return make_error(speech,"Reached 5-second timeout on socket flush, WTF.\n"); } if(FD_ISSET(ss->s, &wsel)) { if(sphinx_swrite(ss,NULL,0) != SPHINX_SUCCESS) return make_error(speech, "Error flushing write buffer.\n"); } if(FD_ISSET(ss->s, &rsel)) { if(sphinx_sread(ss, speech) != SPHINX_SUCCESS) return make_error(speech, "Error flushing read buffer.\n"); } } } return SPHINX_SUCCESS; } int sphinx_write(struct ast_speech *speech, void *data, int len) { // ast_log(LOG_NOTICE,"sphinx_write called with %d bytes data\n", len); if(speech->data == NULL) { ast_log(LOG_ERROR, "Socket data does not exist.\n"); ast_speech_change_state(speech,AST_SPEECH_STATE_NOT_READY); return -1; } struct sphinx_state *ss = (struct sphinx_state *)speech->data; if(speech->state == AST_SPEECH_STATE_DONE || ss->final) { // ast_log(LOG_NOTICE, "Not accepting data because already in state DONE\n"); len = 0; } if(ss->s == 0) { ast_log(LOG_ERROR, "Socket does not exist.\n"); ast_speech_change_state(speech,AST_SPEECH_STATE_NOT_READY); return -1; } /* The Sphinx system doesn't seem be helpful in detecting silence and determing * the end of an utterance on its own, so here we use Asterisk's silence detection * DSP to fake sane behaviour. * The Asterisk Generic Speech API strips the frame away from the data we are * sent, so to use the DSP, here we must re-create a frame. */ struct ast_frame f; f.data = data; f.datalen = len; f.samples = len/2; f.mallocd = 0; f.frametype = AST_FRAME_VOICE; f.subclass = AST_FORMAT_SLINEAR; int totalsil; int silence = ast_dsp_silence(ss->dsp, &f, &totalsil); //ast_log(LOG_NOTICE, "DETECT SILENCE: %s, %06d ms\n", silence ? "true" : "false", totalsil); if(!ss->heardspeech && !silence) { ss->noiseframes++; if(ss->noiseframes > SPHINX_NOISE_FRAMES) { // ast_log(LOG_NOTICE, "Detected speech.\n"); ss->heardspeech = 1; ss->noiseframes = 0; speech->flags |= AST_SPEECH_QUIET; speech->flags |= AST_SPEECH_SPOKE; } } else if(ss->heardspeech && silence && totalsil > SPHINX_SILENCE_TIME) { // ast_log(LOG_NOTICE, "Detected %d finishing silence.\n", totalsil); // sending 0 bytes in a DATA request is another way to wrap-up. len = 0; } else if(silence) ss->noiseframes = 0; //ast_log(LOG_NOTICE,"Writing %d bytes to header containing value %d\n", sizeof(int), mylen); struct sphinx_request sr; sr.dlen = len; sr.rtype = REQTYPE_DATA; sr.data = data; int finish = 0; if(sr.dlen == 0) finish = 1; if(sphinx_comm(&sr, speech, finish) != SPHINX_SUCCESS) { ast_log(LOG_ERROR, "Comms error, changing state to NOT_READY\n"); ast_speech_change_state(speech,AST_SPEECH_STATE_NOT_READY); return -1; } return 0; } int sphinx_dtmf(struct ast_speech *speech, const char *dtmf) { ast_log(LOG_DEBUG,"sphinx_dtmf called with %s\n", dtmf); return 0; } int sphinx_start(struct ast_speech *speech) { ast_log(LOG_DEBUG,"sphinx_start called - changing to ready state\n"); if(reinit_speech_data(speech) != SPHINX_SUCCESS) { ast_speech_change_state(speech, AST_SPEECH_STATE_NOT_READY); ast_log(LOG_ERROR, "Cannot reinit speech object, setting NOT READY\n"); return -1; } ast_speech_change_state(speech, AST_SPEECH_STATE_READY); return 0; } int sphinx_change(struct ast_speech *speech, char *name, const char *value) { ast_log(LOG_DEBUG,"sphinx_change called name %s val %s\n", name, value); return 0; } void log_state(struct ast_speech * speech) { switch(speech->state) { case AST_SPEECH_STATE_NOT_READY: ast_log(LOG_NOTICE, "STATE IS NOT_READY\n"); break; case AST_SPEECH_STATE_READY: ast_log(LOG_NOTICE, "STATE IS READY\n"); break; case AST_SPEECH_STATE_WAIT: ast_log(LOG_NOTICE, "STATE IS WAIT\n"); break; case AST_SPEECH_STATE_DONE: ast_log(LOG_NOTICE, "STATE IS DONE\n"); break; } } int sphinx_change_results_type(struct ast_speech *speech, enum ast_speech_results_type results_type) { ast_log(LOG_DEBUG,"sphinx_change_results_type called\n"); return -1; } struct ast_speech_result* sphinx_get(struct ast_speech *speech) { ast_log(LOG_DEBUG,"sphinx_get called\n"); if(speech != NULL) { if(speech->results != NULL) { return speech->results; } } // ast_log(LOG_NOTICE, "sphinx_get called but no results to return.\n"); return NULL; } int sphinx_connect(struct ast_speech *speech, char const *host, const int port) { struct sockaddr_in sin; struct hostent *hp; struct ast_hostent ahp; struct sphinx_state *ss=(struct sphinx_state *)speech->data; /* State checking */ if(speech == NULL) return SPHINX_ERROR; if(sphinx_disconnect(speech) != SPHINX_SUCCESS) return SPHINX_ERROR; if(ss == NULL) return SPHINX_ERROR; if(ss->s != 0) { ast_log(LOG_ERROR, "Socket exists.\n"); return SPHINX_ERROR; } /* Create socket */ hp = ast_gethostbyname(host, &ahp); if (!hp) { ast_log(LOG_ERROR, "Unable to locate host '%s'\n", host); return SPHINX_ERROR; } ss->s = socket(AF_INET, SOCK_STREAM, 0); if (ss->s <= 0) { ast_log(LOG_ERROR, "Unable to create socket: %s\n", strerror(errno)); ss->s = 0; return SPHINX_ERROR; } /* Make connection */ sin.sin_family = AF_INET; sin.sin_port = htons(port); memcpy(&sin.sin_addr, hp->h_addr, sizeof(sin.sin_addr)); if (connect(ss->s, (struct sockaddr *)&sin, sizeof(sin)) && (errno != EINPROGRESS)) { ast_log(LOG_ERROR, "Connect failed with unexpected error: %s\n", strerror(errno)); close(ss->s); ss->s = 0; return SPHINX_ERROR; } ast_log(LOG_DEBUG, "Connect to %s:%d completed.\n", host, port); // No need to get messy with non-blocking, now that we're connected we'll get that rolling if(sphinx_set_blocking(ss->s, 0) != SPHINX_SUCCESS) { close(ss->s); ss->s = 0; return make_error(speech, "Cannot set blocking mode.\n"); } return SPHINX_SUCCESS; } int reinit_speech_data(struct ast_speech *speech) { ast_log(LOG_DEBUG,"initalizing speech data.\n"); if(speech == NULL) return SPHINX_ERROR; if(speech->data == NULL) { speech->data = ast_calloc(sizeof(struct ast_speech),1); if(speech->data == NULL) return SPHINX_ERROR; } struct sphinx_state *ss = (struct sphinx_state *)speech->data; if(ss->dsp != NULL) { ast_dsp_free(ss->dsp); ss->dsp = NULL; } if(ss->preads || ss->pwbytes || ss->prbytes) { ast_log(LOG_ERROR, "Pending reads: %d, bytes in current read: %d - WE DO NOT EXPECT PENDING READS HERE!\n", ss->preads, ss->pwbytes); // TODO: handle this case better. } ss->heardspeech = 0; ss->noiseframes = 0; ss->final = 0; ss->prbytes = 0; ss->pwbytes = 0; ss->preads = 0; ss->rbufused = 0; ss->dsp = ast_dsp_new(); if(ss->dsp == NULL) { ast_log(LOG_ERROR,"Unable to create silence detection DSP\n"); free(ss); speech->data = NULL; return SPHINX_ERROR; } ast_dsp_set_threshold(ss->dsp, SPHINX_SILENCE_THRESHOLD); if(ss->rbuf != NULL) { free(ss->rbuf); ss->rbuf = NULL; } if(ss->sbuf != NULL) { free(ss->sbuf); ss->sbuf = NULL; } ss->rbuf = ast_calloc(SPHINX_BUFSIZE, 1); if(ss->rbuf == NULL) { ast_dsp_free(ss->dsp); ss->dsp = NULL; free(ss); speech->data = NULL; return SPHINX_ERROR; } ss->sbuf = ast_calloc(SPHINX_BUFSIZE, 1); if(ss->sbuf == NULL) { ast_dsp_free(ss->dsp); ss->dsp = NULL; free(ss->rbuf); ss->rbuf = NULL; free(ss); speech->data = NULL; return SPHINX_ERROR; } return SPHINX_SUCCESS; //ast_log(LOG_NOTICE,"initalizing speech data complete.\n"); } int destroy_speech_data(struct ast_speech *speech) { ast_log(LOG_DEBUG,"destroying speech data.\n"); if(speech == NULL) return SPHINX_ERROR; if(speech->data == NULL) return SPHINX_SUCCESS; if(sphinx_disconnect(speech) != SPHINX_SUCCESS) return SPHINX_ERROR; struct sphinx_state *ss = (struct sphinx_state *)speech->data; if(ss->rbuf != NULL) { free(ss->rbuf); ss->rbuf = NULL; } if(ss->sbuf != NULL) { free(ss->sbuf); ss->sbuf = NULL; } if(ss->dsp != NULL) { ast_dsp_free(ss->dsp); ss->dsp = NULL; } free(ss); speech->data = NULL; return SPHINX_SUCCESS; } int sphinx_disconnect(struct ast_speech *speech) { //ast_log(LOG_NOTICE,"disconnecting\n"); struct sphinx_state *ss = (struct sphinx_state *)speech->data; if(ss == NULL) return SPHINX_SUCCESS; if(ss->s != 0) { close(ss->s); } ast_log(LOG_DEBUG, "DISCONNECTED\n"); return SPHINX_SUCCESS; } AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Sphinx Speech Engine");