@InProceedings{Möller2022_1147,
<br/>   author = {Sebastian Möller and Stefan Hillmann and Thilo Michael and Jan Nehring and Tim Polzehl},
<br/>   booktitle = {Studientexte zur Sprachkommunikation: Elektronische Sprachsignalverarbeitung 2022},
<br/>   title = {Upcoming new ITU-T recommendation on the evaluation of text-based chatbots},
<br/>   year = {2022},
<br/>   editor = {Oliver Niebuhr and Malin Svensson Lundmark and Heather Weston},
<br/>   month = mar,
<br/>   pages = {97--104},
<br/>   publisher = {TUDpress, Dresden},
<br/>   abstract  = {The evaluation of spoken dialog systems has been an object of scientific 
research for decades. Whereas standardized methods were made available by the International Telecommunication Union (ITU-T), a comparable level of maturity is still 
missing for the evaluation of text-based chatbots. This contribution presents ongoing 
work developing a new ITU-T Recommendation describing subjective evaluation 
methods to quantify the quality of services relying on text-based chatbots, as experienced by the users of such services. Chatbots addressed by the upcoming Recommendation enable a text-based natural language interaction with a human user via a text 
interface on a turn-by-turn basis. They possess natural language understanding, dialogue management, and natural language generation capabilities. The evaluation methods address different aspects of quality from a user’s point of view, taking the chatbot 
as a black box. They are based on laboratory or remote experiments in which participants interact with the chatbot in order to perform a pre-defined, realistic task. The 
participant’s opinion on perceptive quality dimensions is solicited with the help of 
questionnaires, and examples of such questionnaires are provided.},
<br/>   isbn = {978-3-95908-548-9},
<br/>   issn = {0940-6832},
<br/>   keywords = {Voice Assistants & Speech Dialogue Systems},
<br/>   url = {https://www.essv.de/pdf/2022_97_104.pdf},
<br/>}