Vanilla JS example

[flatsiedatsie]

It would be great if there was a minimal vanilla JS example of how to use this in a project. I don't use React, so the current output of npm install is very difficult to extract any understanding from.

[xenova]

Check out the whisper-tiny model card for example usage of the model: https://huggingface.co/Xenova/whisper-tiny.en

You'll need to implement the additional UI features and functionality yourself (e.g., mic input), though. But there are resources online to assist you!

Example: Transcribe English.

// npm i @xenova/transformers
import { pipeline } from '@xenova/transformers';

let url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav';

// Create translation pipeline
let transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
let output = await transcriber(url);
// { text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." }

Example: Transcribe English w/ timestamps.

// npm i @xenova/transformers
import { pipeline } from '@xenova/transformers';

let url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav';

// Create translation pipeline
let transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
let output = await transcriber(url, { return_timestamps: true });
// {
//   text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country."
//   chunks: [
//     { timestamp: [0, 8],  text: " And so my fellow Americans ask not what your country can do for you" }
//     { timestamp: [8, 11], text: " ask what you can do for your country." }
//   ]
// }

Example: Transcribe English w/ word-level timestamps.

// npm i @xenova/transformers
import { pipeline } from '@xenova/transformers';

let url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav';

// Create translation pipeline
let transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
let output = await transcriber(url, { return_timestamps: 'word' });
// {
//   "text": " And so my fellow Americans ask not what your country can do for you ask what you can do for your country.",
//   "chunks": [
//     { "text": " And", "timestamp": [0, 0.78] },
//     { "text": " so", "timestamp": [0.78, 1.06] },
//     { "text": " my", "timestamp": [1.06, 1.46] },
//     ...
//     { "text": " for", "timestamp": [9.72, 9.92] },
//     { "text": " your", "timestamp": [9.92, 10.22] },
//     { "text": " country.", "timestamp": [10.22, 13.5] }
//   ]
// }

[flatsiedatsie]

Thanks!

In the end I extracted the necessary part from the React version.

whisper_worker.js

import { pipeline,env } from './js/transformers.js';
console.log("WHISPER WEB WORKER EXISTS");

env.allowLocalModels = false;

addEventListener('message', async (event) => {
	console.log("WHISPER WEB WORKER: RECEIVED MESSAGE");
	console.log("WHISPER WEB WORKER: event.data: ", event.data);
	
    const message = event.data;
	let task = message.task;

    // Do some work...
    // TODO use message data
	try{
		
	    let transcript = await transcribe(
	        message.task.recorded_audio,
	        message.model,
	        message.multilingual,
	        message.quantized,
	        message.subtask,
	        message.language,
	    );
		console.log("WHISPER WEB WORKER: TRANSCRIPTION RESULT: ", transcript);
	    if (transcript === null){
	    	console.error("WHISPER WEB WORKER: transcription was null");
	    }
	    if (typeof transcript === 'undefined'){
	    	console.error("WHISPER WEB WORKER: transcription was undefined??");
	    }

		delete task.recorded_audio;
		task['transcript'] = transcript;
	    // Send the result back to the main thread
	    self.postMessage({
			task: task,
	        status: "complete",
	        //task: "automatic-speech-recognition",
	        transcript: transcript,
	    });
		
	}catch(e){
		console.error("ERROR: whisper worker: ", e);
	}
    

});



// Define model factories
// Ensures only one model is created of each type

class PipelineFactory {
    static task = null;
    static model = null;
    static quantized = null;
    static instance = null;

    constructor(tokenizer, model, quantized) {
        this.tokenizer = tokenizer;
        this.model = model;
        this.quantized = quantized;
    }

    static async getInstance(progress_callback = null) {
        if (this.instance === null) {
            this.instance = pipeline(this.task, this.model, {
                quantized: this.quantized,
                progress_callback,

                // For medium models, we need to load the `no_attentions` revision to avoid running out of memory
                revision: this.model.includes("/whisper-medium") ? "no_attentions" : "main"
            });
        }

        return this.instance;
    }
}


class AutomaticSpeechRecognitionPipelineFactory extends PipelineFactory {
    static task = "automatic-speech-recognition";
    static model = null;
    static quantized = null;
}





const transcribe = async (
    audio,
    model,
    multilingual,
    quantized,
    subtask,
    language,
) => {
	console.log("in transcribe. audio: ", audio);
	console.log("whisper web worker: in transcribe.  model,multilingual,quantized,subtask,language: ", model, multilingual, quantized, subtask, language);
    

	let output = null;

	try{
		const isDistilWhisper = model.startsWith("distil-whisper/");
		
	    let modelName = model;
	    if (!isDistilWhisper && !multilingual) {
	        modelName += ".en"
	    }
		
	    const p = AutomaticSpeechRecognitionPipelineFactory;
	    if (p.model !== modelName || p.quantized !== quantized) {
	        // Invalidate model if different
	        p.model = modelName;
	        p.quantized = quantized;

	        if (p.instance !== null) {
	            (await p.getInstance()).dispose();
	            p.instance = null;
	        }
	    }
		
	    // Load transcriber model
	    let transcriber = await p.getInstance((data) => {
			console.log("whisper web worker: posting something back: ", data);
	        self.postMessage(data);
	    });

	    const time_precision =
	        transcriber.processor.feature_extractor.config.chunk_length /
	        transcriber.model.config.max_source_positions;

	    // Storage for chunks to be processed. Initialise with an empty chunk.
	    let chunks_to_process = [
	        {
	            tokens: [],
	            finalised: false,
	        },
	    ];

	    // TODO: Storage for fully-processed and merged chunks
	    // let decoded_chunks = [];
		
		
		
	    function chunk_callback(chunk) {
			console.log("in whisper chunk callback. chunk: ", chunk);
	        let last = chunks_to_process[chunks_to_process.length - 1];

	        // Overwrite last chunk with new info
	        Object.assign(last, chunk);
	        last.finalised = true;

	        // Create an empty chunk after, if it not the last chunk
	        if (!chunk.is_last) {
	            chunks_to_process.push({
	                tokens: [],
	                finalised: false,
	            });
	        }
	    }
		
		
		
	    // Inject custom callback function to handle merging of chunks
	    function callback_function(item) {
			//console.log("whisper_worker: COMPLETE?  item: ", item);
	        let last = chunks_to_process[chunks_to_process.length - 1];

	        // Update tokens of last chunk
	        last.tokens = [...item[0].output_token_ids];

	        // Merge text chunks
	        // TODO optimise so we don't have to decode all chunks every time
	        let data = transcriber.tokenizer._decode_asr(chunks_to_process, {
	            time_precision: time_precision,
	            return_timestamps: true,
	            force_full_sequences: false,
	        });

	        self.postMessage({
	            status: "update",
	            //task: "automatic-speech-recognition",
	            data: data,
	        });
	    }
		
	    // Actually run transcription
	    output = await transcriber(audio, {
			
	        // Greedy
	        top_k: 0,
	        do_sample: false,

	        // Sliding window
	        chunk_length_s: isDistilWhisper ? 20 : 30,
	        stride_length_s: isDistilWhisper ? 3 : 5,

	        // Language and task
	        language: language,
	        task: subtask,

	        // Return timestamps
	        return_timestamps: true,
	        force_full_sequences: false,

	        // Callback functions
	        callback_function: callback_function, // after each generation step
	        chunk_callback: chunk_callback, // after each chunk is processed
	    })

		.catch((error) => {
			console.error("ERROR, actually running whisper failed");

	        return null;
	    });
		
		console.log("beyond WHISPER transcribe. output: ", output);
		
	}
	catch(e){
		console.error("Whisper worker: error in transcribe function: ", e);
	}
	

    return output;
};

Which I start with this code in a JS module:

window.whisper_worker = null;
window.whisper_worker_busy = false;
let whisper_worker_error_count = 0;







//
//   WHISPER
//

function create_whisper_worker(){
	console.log("in create_whisper_worker");
	
	window.whisper_worker = null;
	window.whisper_worker = new Worker('./whisper_worker.js', {
	  type: 'module'
	});

	console.log("whisper_module: window.whisper_worker: ", window.whisper_worker);
	
	window.whisper_worker.addEventListener('message', e => {
		//console.log("whisper_module: received message from whisper_worker: ", e.data);

		
		if(typeof e.data.status == 'string'){
			if(e.data.status == 'progress'){
				//console.log("whisper worker sent download percentage: ", e.data.progress);
				let whisper_progress_el = document.getElementById('download-progress-whisper');
				if(whisper_progress_el == null){
					console.error("whisper (down)load progress element is missing");
					add_chat_message("whisper",'download_progress#setting---');
				}
				else{
					//console.log("updating whisper (down)load progress");
					whisper_progress_el.value = e.data.progress / 100;
				}
				
			}
			else if(e.data.status == 'ready'){
				console.log("whisper worker sent ready message");
				window.whisper_worker_busy = false;
				add_chat_message("whisper",get_translation('Voice_recognition_has_loaded'));
				let whisper_progress_el = document.getElementById('download-progress-whisper');
				if(whisper_progress_el){
					whisper_progress_el.classList.add('download-complete-chat-message');
				}
				else{
					console.error("whisper became ready, but cannot find loading progress indicator element");
				}
			}
			else if(e.data.status == 'initiate'){
				console.log("whisper worker sent initiate message");
			}
			else if(e.data.status == 'download'){
				console.log("whisper worker sent download message");
				add_chat_message("whisper","(down)loading: " + e.data.file);
			}
			
			else if(e.data.status == 'update'){
				if(typeof e.data.data == 'object' && e.data.data != null && e.data.data.length){
					set_chat_status(e.data.data[0],2);
				}
				
			}
			
			else if(e.data.status == 'complete'){
				window.whisper_worker_busy = false;
				console.log('GOT WHISPER COMPLETE.  e.data: ', e.data);
				console.log('GOT WHISPER COMPLETE.  e.data.transcript: ', e.data.transcript);
				console.log('GOT WHISPER COMPLETE.  e.data.task: ', e.data.task);
				
				if(e.data.transcript == null){
					console.warn("whisper recognition failed. If this is the first run, that's normal.");
					set_state(LISTENING);
				}
				else if(typeof e.data.transcript != 'undefined'){
					console.log("whisper returned transcription text: ", e.data.transcript);
					
					if(Array.isArray(e.data.transcript)){
						console.log("typeof transcription is array");
					}
					else if(typeof e.data.transcript == 'object'){
						if(typeof e.data.transcript.text == 'string'){
							console.log("GOT TEXT: ", e.data.transcript.text);
						}
					}
				}
				else{
					console.log("transcript was not in whisper e.data");
				}
				
				//add_chat_message("whisper","(down)loading: " + e.data.file);
			}
			else{
				console.log("whisper worker sent a content message");
				window.whisper_worker_busy = false;
				
				if(e.data.data == null){
					console.warn("whisper recognition failed. If this is the first run, that's normal.");
					set_state(LISTENING);
				}
			}
		}
			
			if(window.enable_microphone == false){
				console.log("whisper worker returned audio file, but in the meantime enable_microphone was disabled. Throwing away the data.");
			}
			else{
				
				/*
			
				if(window.whisper_queue.length){
					console.log("whisper worker done, but there is more work to do. Sentences still in whisper_queue: ", window.whisper_queue.length);
					let next_sentence = window.whisper_queue[0][0] + window.whisper_queue[0][1]; // sentence plus punctuation mark
					window.whisper_queue.splice(0,1);
				
				
					whisper_worker.postMessage({'whisper_counter':window.whisper_counter,'sentence':next_sentence});
					window.whisper_counter++;
				}
				else{
					console.log("whisper worker was done, and there are no more sentences in the whisper queue. Worker is now idle.");
					window.whisper_worker_busy = false;
				}
				*/
			}
	
	});


	window.whisper_worker.addEventListener('error', (error) => {
		console.error("ERROR: whisper_worker sent error. terminating!. Error was: ", error, error.message);
		whisper_worker_error_count++;
		
		window.whisper_worker.terminate();
		window.whisper_worker_busy = false;
		if(typeof error != 'undefined' && whisper_worker_error_count < 10){
			setTimeout(() => {
				console.log("attempting to restart whisper worker");
				create_whisper_worker();
			},1000);
		}
		else{
			console.error("whisper_worker errored out");
		}
	});
}

// create whisper worker
create_whisper_worker();


//
//  Send audio buffer to whisper worker
//
function do_whisper_web(task,language=null){
	console.log("in do_whisper_web. task: ", task);
	
	if(window.whisper_worker_busy){
		console.error("do_whisper_web was called while whisper worker was busy. Aborting.");
		return
	}
	
	if(typeof task.recorded_audio == 'undefined'){
		console.error("do_whisper_web: task did not contain recorded_audio. Aborting.");
		return
	}
	
	task.state == 'stt_in_progress';
	
	let multilingual = false;
	if(typeof language == 'string'){
		if(language != 'en'){
			multilingual = true;
		}
	}
	const quantized = false;
	const model = "Xenova/whisper-tiny";
	
	const subtask = null;
	
	console.log("do_whisper_web: sending audio to whisper worker: ", task.recorded_audio);
	
    window.whisper_worker.postMessage({
        task:task,
        model,
        multilingual,
        quantized,
        subtask: multilingual ? subtask : null,
        language:
            multilingual && language !== "auto" ? language : null,
    });

}
window.do_whisper_web = do_whisper_web;

[hadfield]

I had a similar desire to use this in vanilla javascript form and had extracted some code from the react version to give it a try, but this version is much nicer. thanks for posting it! A rough version of whisper + vanilla js is in the repo:
https://github.com/vital-ai/vital-stt-js
i wanted to test this out in combination with a browser based wake word detector, which I posted here:
https://github.com/chat-ai-app/chat-ai-assistant-demo
with webapp posted here:
https://demo-voice.chat.ai/