// This is a little quick and dirty free tool to get a complete spectrogram of an mp3 song in a // form that's easy to import into other programs. It is for those who are doing non-realtime // video animation synchronized to music who need spectral data for an entire song in an easy to // read and use, video-friendly form. // This application plays back an mp3 audio file while sampling the Flash computeSpectrum() // function once for every frame of video (you can select your own frame rate). Since it gets // its spectrograms in real time it's only really useful and accurate for this purpose if your // computer can keep up. If it can't keep up, it will do its best to fill in the blanks with // the most recent successful spectrum reading. // After the audio playback is complete, the entire spectrogram is written out to a comma-delimited // plain text file where each line in the text file is a list of the power levels in each of 256 // bands for a single video frame. The power levels are floating-point values between 0 and 1. // The line number in the text file corresponds to the frame number in the video. The text file is // named "MP3_Spectrum.csv" and is stored in the users "Desktop" directory. // On my graphics card and PC, I'm able to draw the spectrogram for each frame without missing // frames, but if you have a slower graphics card or computer, you may want to disable the drawing // of the spectrum by clicking on the black "Show Spectrum: Yes/No" button in the middle. // When I get a little more time and experience I'll do a non-real time version of this that is // guaranteed to sample every frame without worrying about the constraints of keeping up with // real-time sampling. For now, I'm just really thankful that Adobe added the computeSpectrum() // function to Flash 10 and above so the not so mathematically inclined among us can finally have // access to this very interesting data. I hope this little utility helps in your music video // creations! // The compiled application is the file that ends in ".air" and can be played back when you install // the Adobe AIR player, which can be downloaded here: http://get.adobe.com/air/ // Ken Meyering, April 4, 2010 // 2/22/2011 Updated to plot average amplitude and put data into a format with column headers that // can be imported into databases such as MySQL. Also added SQL script to load data into multiple // MySQL tables. package { import flash.events.MouseEvent; import flash.display.Sprite; import flash.external.* ; import flash.media.* ; import flash.display.* ; import flash.events.* ; import flash.net.* ; import flash.utils.*; import flash.filesystem.* ; import flash.text.TextField; import flash.text.TextFieldType; import flash.text.TextFieldAutoSize; import flash.text.TextFormat; import flash.desktop.NativeApplication; import fl.motion.Color; public class MP3_Spectrum_to_Text_File extends MovieClip { var myFrameRate = 30; var volumeAveragingWindow = 7; var spectrumAveragingWindow = 2; var drawSpectrum: Boolean = true; var drawVolume: Boolean = true; var myGraph:MovieClip=new MovieClip(); var outputCSV: String = "MP3_Spectrum.csv"; // This file will be saved to the Desktop var outputSQL: String = "MP3_Spectrum.sql"; // This file will be saved to the Desktop var music: Sound = new Sound(); var soundURL: String; var fileFilter: FileFilter = new FileFilter("MP3 Audio Files", "*.mp3"); var soundChannel: SoundChannel = new SoundChannel(); var sound: Sound; var videoFrameNumber: Number = 0; var lastPosition: Number = -1; var thisPosition: Number = 0; var soundStarted: Boolean = false; var soundFinished: Boolean = false; var writingFileMessageDisplayed = false; var fileWritten: Boolean = false; var inputFile: File = new File(); var startTime: Number; var txtMessage: TextField = new TextField(); var txtFrameRate: TextField = new TextField(); var format: TextFormat = new TextFormat(); var showPath: TextField = new TextField(); var btnSpectrumYes=new SpectrumYes(); var btnSpectrumNo=new SpectrumNo(); var btnStart=new Start(); var min = 100; var max = 0; var peakNarrowBand = 0; var lastAverage: Number = 0; var totalFrameCount; var thisStageWidth = stage.stageWidth; var thisStageHeight = stage.stageHeight; var spectrumTopY; var spectrumBottomY; var spectrumHeight; var volumeTopY; var volumeBottomY; var volumeHeight; var RED = 0xFF0000; var GREEN = 0x00AA00; var BLUE = 0x0000FF; var WHITE = 0xFFFFFF; var spectrumWidth = 256; var volumeDivisor = Math.sqrt(spectrumWidth); var peakSpectrumValue = 0; var myByteArray = new ByteArray(); var myRawSpectrumArray = new Array(); var mySpectrumAverageArray = new Array(); var myVolumeArray = new Array(); var myAverageVolumeArray = new Array(); var narrowBandArray = new Array(); var averageNarrowBandArray = new Array(); var myCSVFile: File = File.desktopDirectory.resolvePath(outputCSV); var mySQLFile: File = File.desktopDirectory.resolvePath(outputSQL); var myCSVPath: String = myCSVFile.nativePath; var mySQLPath: String = mySQLFile.nativePath; var arraysPopulated=false; var CSVWritten=false; var SQLWritten=false; var currentInterval:Number; public function MP3_Spectrum_to_Text_File() { startUp(); } private function startUp() { format.font = "Arial"; format.color = 0x000000; format.size = 25; format.align = "center"; txtMessage.defaultTextFormat = format; txtMessage.x = 0; txtMessage.y = 10; txtMessage.width = thisStageWidth; txtMessage.height = 30; txtMessage.autoSize = "center"; txtMessage.text = "Please enter the output frame rate (FPS):"; showPath.width = 0; showPath.x = thisStageWidth / 2; showPath.y = 40; showPath.height = 50; showPath.autoSize = "center"; showPath.defaultTextFormat = format; showPath.text = ""; txtFrameRate.defaultTextFormat = format; txtFrameRate.x = thisStageWidth / 2 - 50; txtFrameRate.y = 80; txtFrameRate.width = 100; txtFrameRate.height = 30;; txtFrameRate.multiline = false; txtFrameRate.background = true; txtFrameRate.backgroundColor = 0xAA0000; txtFrameRate.wordWrap = false; txtFrameRate.textColor = 0xFFFFFF; txtFrameRate.text = myFrameRate; txtFrameRate.type = TextFieldType.INPUT; addChild(btnSpectrumYes); addChild(btnSpectrumNo); addChild(btnStart); btnSpectrumYes.x=thisStageWidth / 2; btnSpectrumYes.y=147; btnSpectrumNo.x=thisStageWidth / 2; btnSpectrumNo.y=147; btnStart.x=thisStageWidth/2; btnStart.y=201; btnSpectrumYes.visible=true; btnSpectrumNo.visible=false; btnStart.visible=true; addChild(showPath); addChild(txtMessage); addChild(txtFrameRate); addChild(myGraph); btnSpectrumYes.addEventListener(MouseEvent.CLICK, toggleShowSpectrum); btnSpectrumNo.addEventListener(MouseEvent.CLICK, toggleShowSpectrum); btnStart.addEventListener(MouseEvent.CLICK, begin); volumeHeight = 20; volumeBottomY = thisStageHeight; volumeTopY = volumeBottomY - volumeHeight; spectrumHeight = 150; spectrumBottomY = volumeTopY - 10; spectrumTopY = spectrumBottomY - spectrumHeight; } private function toggleShowSpectrum(evt: Event): void { drawSpectrum = !drawSpectrum; if (drawSpectrum) { btnSpectrumYes.visible=true; btnSpectrumNo.visible=false; } else { btnSpectrumYes.visible=false; btnSpectrumNo.visible=true; } } private function begin(evt: Event): void { soundStarted=false; arraysPopulated=false; CSVWritten=false; SQLWritten=false; min=0; max=0; showPath.text = ""; myByteArray = new ByteArray(); myRawSpectrumArray = new Array(); mySpectrumAverageArray = new Array(); myVolumeArray = new Array(); myAverageVolumeArray = new Array(); narrowBandArray = new Array(); averageNarrowBandArray = new Array(); removeChild(btnStart); txtMessage.text = "Please select an MP3 file..."; myFrameRate = Number(txtFrameRate.text); stage.frameRate = myFrameRate; inputFile.resolvePath(File.desktopDirectory.nativePath); inputFile.browseForOpen("Please select an MP3 file to analyze.", [fileFilter]); inputFile.addEventListener(Event.SELECT, fileSelected); inputFile.addEventListener(Event.CANCEL, applicationExit); } private function fileSelected(evt: Event): void { inputFile.removeEventListener(Event.SELECT, fileSelected); soundURL = inputFile.nativePath; sound = new Sound(new URLRequest(soundURL)); sound.addEventListener(ProgressEvent.PROGRESS, cachingAudio); sound.addEventListener(Event.COMPLETE, startPlaying); txtMessage.text = "Caching audio..."; } private function cachingAudio(evt: ProgressEvent): void { var loadingProgress = evt.bytesLoaded / evt.bytesTotal; showLoadProgress(loadingProgress); } private function startPlaying(evt: Event): void { txtMessage.text = "Analyzing audio and storing spectrums..."; sound.removeEventListener(Event.COMPLETE, startPlaying); soundChannel = sound.play(0); soundChannel.addEventListener(Event.SOUND_COMPLETE, sound_complete); addEventListener(Event.SOUND_COMPLETE, sound_complete); sound.addEventListener(Event.SOUND_COMPLETE, sound_complete); addEventListener(Event.ENTER_FRAME, on_enter_frame); } private function on_enter_frame(e: Event): void { clearGraph(); if(!soundFinished) buildSpectrum(); if(soundFinished && !arraysPopulated) { txtMessage.text = "Sound Complete. Populating all arrays..."; removeEventListener(Event.ENTER_FRAME,on_enter_frame); currentInterval=setInterval(populateAllArrays,1000); } if(soundFinished && arraysPopulated && !CSVWritten) { txtMessage.text = "Writing CSV File..."; showPath.text = myCSVPath; removeEventListener(Event.ENTER_FRAME,on_enter_frame); currentInterval=setInterval(writeCSV,1000); } if(soundFinished && arraysPopulated && CSVWritten && !SQLWritten) { txtMessage.text = "Writing SQL File..."; showPath.text = mySQLPath; removeEventListener(Event.ENTER_FRAME,on_enter_frame); currentInterval=setInterval(writeSQL,1000); } } private function buildSpectrum(): void { var thisGraph = new Array(); var duration = (getTimer() - startTime) / 1000; var sum = 0; videoFrameNumber = Math.round(duration * myFrameRate); if (thisPosition == 0) thisPosition = soundChannel.position / 1000; if (thisPosition > 0) { if (!soundStarted) { startTime = getTimer(); videoFrameNumber = 0; soundStarted = true; } if (!soundFinished) { if (videoFrameNumber > 0) { showPath.text=getTimeCode(videoFrameNumber); SoundMixer.computeSpectrum(myByteArray, true, 0); for (var i: uint = 0;i < spectrumWidth;i++) { var thisValue = myByteArray.readFloat(); thisGraph[i] = thisValue; if (thisValue > peakSpectrumValue) peakSpectrumValue = thisValue; } myRawSpectrumArray[videoFrameNumber] = thisGraph; var thisVolume = getVolume(thisGraph); myVolumeArray[videoFrameNumber] = thisVolume; var averageVolume = getAverageVolume(videoFrameNumber); myAverageVolumeArray[videoFrameNumber] = averageVolume; if (drawVolume) DrawVolume(averageVolume); if (drawSpectrum) plotSpectrum(getSpectrumAverage(videoFrameNumber)); } } } return; } private function getVolume(thisGraph) { var sum = 0; for (var i: uint = 0;i < thisGraph.length;i++) { var thisValue = thisGraph[i]; var thisLevel = Math.abs(thisValue); sum += thisLevel; } var vol = sum / volumeDivisor; if (vol < min) min = vol; if (vol > max) max = vol; return vol; } private function sound_complete(evt: Event) { soundFinished=true; soundChannel.removeEventListener(Event.SOUND_COMPLETE, sound_complete); } private function getTime(frame) { return frame / myFrameRate; } private function trimEnd(myString) { return myString.substring(0, myString.length - 1); } private function populateAverageVolumeArray() { txtMessage.text = "Populating average volume array..."; addChild(txtMessage); for (var i: Number = 0;i < totalFrameCount;i++) //get average levels for smooth graph { if(!myAverageVolumeArray[i]) myAverageVolumeArray[i] = getAverageVolume(i); } return; } private function populateAverageSpectrumArray() { txtMessage.text = "Populating average spectrum array..."; for (var i: Number = 0;i < totalFrameCount;i++) //get average levels for smooth graph { mySpectrumAverageArray[i] = getSpectrumAverage(i); } return; } private function drawRectangle(x1, y1, x2, y2, thisColor) { var thisRectangle = new Sprite(); thisRectangle.graphics.lineStyle(1, thisColor); thisRectangle.graphics.beginFill(thisColor, 1); thisRectangle.graphics.drawRect(x1, y1, x2 - x1, y2 - y1); thisRectangle.graphics.endFill(); myGraph.addChild(thisRectangle); return; } private function populateAllArrays() { clearInterval(currentInterval); totalFrameCount = myVolumeArray.length; populateMyRawSpectrumArray(); populateAverageVolumeArray(); //populateAverageSpectrumArray(); arraysPopulated=true; addEventListener(Event.ENTER_FRAME,on_enter_frame); } private function applicationExit(evt: Event) { var exitingEvent = new Event(Event.EXITING, false, true); NativeApplication.nativeApplication.dispatchEvent(exitingEvent); if (!exitingEvent.isDefaultPrevented()) { NativeApplication.nativeApplication.exit(); } } private function DrawVolume(thisVolume): void { var volumeWidth = thisStageWidth * thisVolume/2; drawRectangle(0, volumeTopY, volumeWidth, volumeBottomY, RED); return; } private function showLoadProgress(thisProgress): void { drawRectangle(0, volumeTopY, thisProgress * thisStageWidth, volumeBottomY, GREEN); return; } private function getSpectrumAverage(videoFrameNumber) { if (spectrumAveragingWindow < 2) { return myRawSpectrumArray[videoFrameNumber]; } var averageSpectrum = new Array(); var startFrame = videoFrameNumber - spectrumAveragingWindow; if (startFrame < 1) startFrame = 1; var samples = videoFrameNumber - startFrame + 1; var lastGraph = myRawSpectrumArray[videoFrameNumber] for (var band = 0; band < spectrumWidth; band++) { var sum = 0; var lastLevel; for (var i = startFrame; i < videoFrameNumber + 1; i++) { var thisGraph; if (myRawSpectrumArray[i]) thisGraph = myRawSpectrumArray[i]; else thisGraph = lastGraph; sum += thisGraph[band]; lastGraph = thisGraph; } var averageLevel = sum / samples; averageSpectrum[band] = averageLevel; } return averageSpectrum; } private function getAverageVolume(videoFrameNumber) { var startFrame = videoFrameNumber - volumeAveragingWindow; if (startFrame < 1) startFrame = 1; var samples = videoFrameNumber - startFrame + 1; var sum = 0; for (var i = startFrame; i < videoFrameNumber + 1; i++) { if (!myVolumeArray[i]) myVolumeArray[i] = lastAverage; sum += myVolumeArray[i]; } var averageLevel = sum / samples; lastAverage = averageLevel; return averageLevel; } private function getTimeCode(frame) { var totalSeconds = frame / myFrameRate; var f = frame % myFrameRate; var h = Math.floor(totalSeconds / 60 / 60); var m = Math.floor(totalSeconds / 60 - h * 60); var s = Math.floor(totalSeconds - h * 60 * 60 - m * 60); h = right("00" + h.toString(), 2); m = right("00" + m.toString(), 2); s = right("00" + s.toString(), 2); f = right("00" + f.toString(), 2); return h + ":" + m + ":" + s + "." + f; } private function NaNCheck(number:Number) { if(isNaN(number)) return 0; return number; } private function right(string, count) { if (count == undefined) var count = 1; var s = string.length - count; var f = string.length; return string.substring(s, f); } private function stopMovie() { removeEventListener(Event.ENTER_FRAME, on_enter_frame); } private function populateMyRawSpectrumArray() { txtMessage.text="Populating raw spectrum array..."; for (var counter = myRawSpectrumArray.length; counter > 0; counter--) { if (!myRawSpectrumArray[counter]) { var dataFound = false; for (var i = counter - 1; i > 0; i--) { if (myRawSpectrumArray[i] && !dataFound) { myRawSpectrumArray[counter] = myRawSpectrumArray[i]; dataFound = true; } } } } return; } private function plotSpectrum(myArray) { var elementCount = myArray.length; var myWidth = thisStageWidth / elementCount; for (var i = 0; i < elementCount; i++) { var thisValue = Math.abs(myArray[i]); if (thisValue) { var thisStartX = i * myWidth; var thisEndX = thisStartX + myWidth; var thisHeight = spectrumHeight * thisValue; var thisTopY = spectrumBottomY - thisHeight; drawRectangle(thisStartX, thisTopY, thisEndX, spectrumBottomY, BLUE); } } return; } private function writeCSV(): void { clearInterval(currentInterval); var myFileOutput = ""; var fieldDelimiter=","; var lineDelimiter="\r\n"; var columns: String = "frame"+fieldDelimiter+"time"+fieldDelimiter+"timecode"+fieldDelimiter+"amplitude"+fieldDelimiter+"average"+fieldDelimiter+""; for (var band = 0; band < spectrumWidth; band++) { var thisBandLabel = band + 1; columns += "band" + thisBandLabel + fieldDelimiter; } myFileOutput += trimEnd(columns) + lineDelimiter; var myFileStream: FileStream = new FileStream(); for (var frame: Number = 1;frame < totalFrameCount;frame++) { columns = ""; columns += frame.toString() + fieldDelimiter; columns += NaNCheck(getTime(frame)) + fieldDelimiter; columns += getTimeCode(frame) + fieldDelimiter; columns += NaNCheck(myVolumeArray[frame]) + fieldDelimiter; columns += NaNCheck(myAverageVolumeArray[frame]) + fieldDelimiter; var thisGraph = myRawSpectrumArray[frame]; for (var k = 0; k < thisGraph.length; k++) { columns += NaNCheck(thisGraph[k]) + fieldDelimiter; } myFileOutput += trimEnd(columns) + lineDelimiter; } try { myFileStream.open(myCSVFile, FileMode.WRITE); myFileStream.writeUTFBytes(myFileOutput); } catch (e: Error) { txtMessage.text = "Unabled to open "+myCSVFile; } CSVWritten=true; addEventListener(Event.ENTER_FRAME,on_enter_frame); } private function writeSQL(): void { clearInterval(currentInterval); var myFileStream: FileStream = new FileStream(); var sql:String=""; myFileStream.open(mySQLFile, FileMode.WRITE); soundURL=soundURL.replace("'","{single_quote}"); try { sql+= "#Create the spectrum database if it doesn't already exist\r\n"+ "CREATE database IF NOT EXISTS spectrums;\r\n"+ "USE spectrums;\r\n"+ "\r\n"+ "#Create the filename table if it doesn't exist\r\n"+ "CREATE TABLE IF NOT EXISTS mp3files (\r\n"+ "id INT UNSIGNED NOT NULL AUTO_INCREMENT ,\r\n"+ "url VARCHAR( 255 ) NOT NULL ,\r\n"+ "spectrum_width INT UNSIGNED NOT NULL,\r\n"+ "frame_rate DOUBLE NOT NULL,\r\n"+ "total_frames INT UNSIGNED NOT NULL,\r\n"+ "length VARCHAR(15),\r\n"+ "PRIMARY KEY ( id )\r\n"+ ");\r\n"+ "\r\n"+ "#Create the volume table if it doesn't exist\r\n"+ "CREATE TABLE IF NOT EXISTS volume (\r\n"+ "id INT UNSIGNED NOT NULL,\r\n"+ "frame INT UNSIGNED NOT NULL,\r\n"+ "raw_volume DOUBLE,\r\n"+ "average_volume DOUBLE,\r\n"+ "PRIMARY KEY (id,frame),\r\n"+ "CONSTRAINT fk_volume_id FOREIGN KEY (id) REFERENCES mp3Files(id) ON DELETE CASCADE);\r\n"+ "\r\n"+ "#Create the spectrum table if it doesn't exist\r\n"+ "CREATE TABLE IF NOT EXISTS spectrum (\r\n"+ "id INT UNSIGNED NOT NULL,\r\n"+ "frame MEDIUMINT UNSIGNED NOT NULL,\r\n"+ "band SMALLINT UNSIGNED NOT NULL,\r\n"+ "level DOUBLE,\r\n"+ "PRIMARY KEY (id,frame,band),\r\n"+ "CONSTRAINT fk_spectrum_id FOREIGN KEY (id) REFERENCES mp3Files(id) ON DELETE CASCADE);\r\n"+ "\r\n"+ "#Clear out the volume and spectrum table for this filename\r\n"+ "DELETE FROM mp3files WHERE url='"+soundURL+"';\r\n"+ "#Add this filename to the table\r\n"+ "INSERT INTO mp3files(url,spectrum_width,frame_rate,total_frames,length) VALUES ('"+soundURL+"',"+spectrumWidth+","+myFrameRate+","+myVolumeArray.length+", '"+getTimeCode(myRawSpectrumArray.length)+"');\r\n"+ "SET @ID=(SELECT id FROM mp3files WHERE url='"+soundURL+"');\r\n"+ "\r\n"+ "#Insert the volume array into the volume table\r\n"+ "INSERT INTO volume(id,frame,raw_volume,average_volume) VALUES\r\n"; myFileStream.writeUTFBytes(mySqlEscape(sql)); sql=""; for(var volumeFrame=1;volumeFrame