Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
No results found
Show changes
Showing
with 1303 additions and 81 deletions
File added
File added
File added
File added
File added
File added
File added
<group>
<0>100</0>
<1>0</1>
<2>44</2>
<3>0</3>
<4>100</4>
<5>0</5>
<6>45</6>
<7>0</7>
<8>100</8>
<9>0</9>
<10>45</10>
<11>0</11>
</group>
#include "ofMain.h"
#include "ofApp.h"
//========================================================================
int main( ){
ofSetupOpenGL(1024,768,OF_WINDOW); // <-------- setup the GL context
// this kicks off the running of my app
// can be OF_WINDOW or OF_FULLSCREEN
// pass in width and height too:
ofRunApp(new ofApp());
}
#include <array>
#include "ofApp.h"
//--------------------------------------------------------------
void ofApp::setup(){
myo.setup();
//gui
probsClear.addListener(this, &ofApp::probsClearPressed);
resetModel.addListener(this, &ofApp::resetModelPressed);
guiTong.setup("tong");
guiTong.add(tong0.setup("1", 50, 0, 100));
guiTong.add(tong1.setup("2", 0, 0, 100));
guiTong.add(tong2.setup("3", 90, 0, 100));
guiTong.add(tong3.setup("4", 0, 0, 100));
guiTong.add(tong4.setup("5", 90, 0, 100));
guiTong.add(tong5.setup("6", 0, 0, 100));
guiTong.add(tong6.setup("7", 0, 0, 100));
guiTong.add(tong7.setup("8", 60, 0, 100));
guiTong.add(tong8.setup("9", 0, 0, 100));
guiTong.add(tong9.setup("10", 90, 0, 100));
guiTong.add(tong10.setup("11", 0, 0, 100));
guiTong.add(tong11.setup("12", 0, 0, 100));
ofColor green(0, 255, 0);
guiTong.setHeaderBackgroundColor(green);
guiThung.setup("thung", "thung", 250, 10);
guiThung.add(thung0.setup("1", 100, 0, 100));
guiThung.add(thung1.setup("2", 0, 0, 100));
guiThung.add(thung2.setup("3", 10, 0, 100));
guiThung.add(thung3.setup("4", 0, 0, 100));
guiThung.add(thung4.setup("5", 60, 0, 100));
guiThung.add(thung5.setup("6", 0, 0, 100));
guiThung.add(thung6.setup("7", 10, 0, 100));
guiThung.add(thung7.setup("8", 0, 0, 100));
guiThung.add(thung8.setup("9", 60, 0, 100));
guiThung.add(thung9.setup("10", 0, 0, 100));
guiThung.add(thung10.setup("11", 30, 0, 100));
guiThung.add(thung11.setup("12", 0, 0, 100));
ofColor yellow(255, 255, 0);
guiThung.setHeaderBackgroundColor(yellow);
guiGeneral.setup("general", "general", 500, 10);
guiGeneral.add(gain.setup("gain", 1., 0., 1.));
guiGeneral.add(modelControl.setup("run model", false));
guiGeneral.add(resetModel.setup("reset model"));
guiGeneral.add(inputDevice.setup("myo", false));
guiGeneral.add(probsClear.setup("clear"));
//This will make life easier later
allSliders.push_back(tong0);
allSliders.push_back(tong1);
allSliders.push_back(tong2);
allSliders.push_back(tong3);
allSliders.push_back(tong4);
allSliders.push_back(tong5);
allSliders.push_back(tong6);
allSliders.push_back(tong7);
allSliders.push_back(tong8);
allSliders.push_back(tong9);
allSliders.push_back(tong10);
allSliders.push_back(tong11);
allSliders.push_back(thung0);
allSliders.push_back(thung1);
allSliders.push_back(thung2);
allSliders.push_back(thung3);
allSliders.push_back(thung4);
allSliders.push_back(thung5);
allSliders.push_back(thung6);
allSliders.push_back(thung7);
allSliders.push_back(thung8);
allSliders.push_back(thung9);
allSliders.push_back(thung10);
allSliders.push_back(thung11);
//RapidLib
trained = false;
//loading samples
saron_sbpl1.load(ofToDataPath("saron-sbpl1.wav"));
saron_sbpl2.load(ofToDataPath("saron-sbpl2.wav"));
saron_sbpl3.load(ofToDataPath("saron-sbpl3.wav"));
saron_sbpl4.load(ofToDataPath("saron-sbpl4.wav"));
saron_sbpl5.load(ofToDataPath("saron-sbpl5.wav"));
ciblon_tong.load(ofToDataPath("drums-ciblon-medium-tong.wav"));
ciblon_thung.load(ofToDataPath("drums-ciblon-medium-thung.wav"));
//maxi Clock
myClock.setTicksPerBeat(4);//This sets the number of ticks per beat
myClock.setTempo(120);// This sets the tempo in Beats Per Minute
//audio setup
sampleRate = 44100;
bufferSize = 512;
ofxMaxiSettings::setup(sampleRate, 2, initialBufferSize);
ofSoundStreamSetup(2,2,this, sampleRate, bufferSize, 4);
}
//--------------------------------------------------------------
void ofApp::update(){
if (inputDevice) { //Only when Myo is toggled on
//Simple gain control
float emg = myo.getDevices()[0]->getEmgSamples()[4];
streamBuf.pushToWindow(double(emg));
gain = streamBuf.rms() * 0.01;
//Machine learning with quarternions
double myoX = myo.getDevices()[0]->getQuaternion().x();
double myoY = myo.getDevices()[0]->getQuaternion().y();
double myoZ = myo.getDevices()[0]->getQuaternion().z();
double myoW = myo.getDevices()[0]->getQuaternion().w();
if (inputDevice) {
if (recordingState > 0) {
trainingExample tempExample;
tempExample.input = { myoX, myoY, myoZ, myoW };
for (int i = 0; i < allSliders.size() ; ++i) {
tempExample.output.push_back(double(allSliders[i]));
}
trainingSet.push_back(tempExample);
} else if (trained && modelControl == 1) {
std::vector<double> inputVec;
inputVec.push_back(myoX);
inputVec.push_back(myoY);
inputVec.push_back(myoZ);
inputVec.push_back(myoW);
std::vector<double> output = myNN.run(inputVec);
for (int i = 0; i < output.size(); ++i) {
allSliders[i] = int(output[i]);
}
}
}
}
}
//--------------------------------------------------------------
void ofApp::exit(){
myo.stop();
}
//--------------------------------------------------------------
void ofApp::draw(){
//ofClear(0);
ofDrawBitmapString(currentBeat, 100, 300);
ofDrawBitmapString("Hold space to record", 200, 300);
guiTong.draw();
guiThung.draw();
guiGeneral.draw();
if (inputDevice) {
ofSetColor(255);
for ( int i=0; i<myo.getDevices().size(); i++ ) {
stringstream s;
s << "id: " << myo.getDevices()[i]->getId() << endl;
s << "which: " << myo.getDevices()[i]->getWhichArm() << endl;
s << "pose: " << myo.getDevices()[i]->getPose() << endl;
s << "accel: ";
s << myo.getDevices()[i]->getAccel().x << ",";
s << myo.getDevices()[i]->getAccel().y << ",";
s << myo.getDevices()[i]->getAccel().z << endl;
s << "gyro: ";
s << myo.getDevices()[i]->getGyro().x << ",";
s << myo.getDevices()[i]->getGyro().y << ",";
s << myo.getDevices()[i]->getGyro().z << endl;
s << "quaternion: ";
s << myo.getDevices()[i]->getQuaternion().x() << ",";
s << myo.getDevices()[i]->getQuaternion().y() << ",";
s << myo.getDevices()[i]->getQuaternion().z() << ",";
s << myo.getDevices()[i]->getQuaternion().w() << endl;
s << "roll/pitch/yaw: ";
s << myo.getDevices()[i]->getRoll() << ",";
s << myo.getDevices()[i]->getPitch() << ",";
s << myo.getDevices()[i]->getYaw() << endl;
s << "raw data: ";
for ( int j=0; j<8; j++ ) {
s << myo.getDevices()[i]->getEmgSamples()[j];
s << ",";
}
s << endl;
ofSetColor(0);
ofDrawBitmapString(s.str(), 10, 400 + i*100);
}
}
}
//--------------------------------------------------------------
void ofApp::probsClearPressed() {
std::cout << "clearing probs" << std::endl;
for (int i = 0; i < allSliders.size(); ++i) {
allSliders[i] = 0;
}
}
//--------------------------------------------------------------
void ofApp::resetModelPressed() {
std::cout << "resetting models" << std::endl;
myNN.reset();
trainingSet.clear();
modelControl = false;
}
//--------------------------------------------------------------
bool eventTest(int prob) {
int testRand = rand() % 100;
if (testRand < prob) {
return true;
}
return false;
}
//--------------------------------------------------------------
void ofApp::audioOut(float * output, int bufferSize, int nChannels) {
//probs = { tong0, tong1, tong2, tong3, tong4, tong5, tong6, tong7, tong8, tong9, tong10, tong11 };
//probs2 = { thung0, thung1, thung2, thung3, thung4, thung5, thung6, thung7, thung8, thung9, thung10, thung11 };
bool beatsTong[12];
bool beats2[12];
for (int i = 0; i < 12; ++i) {
beatsTong[i] = eventTest(allSliders[i]);
beats2[i] = eventTest(allSliders[i + 12]);
}
int lastCount = 0;
int testMe = 0;
for (int i = 0; i < bufferSize; i++){
myClock.ticker();
if (myClock.tick) {
if (beatsTong[currentBeat]) {
ciblon_tong.trigger();
}
if (beats2[currentBeat]) {
ciblon_thung.trigger();
}
currentBeat = (currentBeat + 1) % 12;
}
outputs[0] = ciblon_tong.playOnce() * gain;
outputs[1] = ciblon_thung.playOnce() * gain;
output[i*nChannels ] = outputs[0];
output[i*nChannels + 1] = outputs[1];
}
}
//--------------------------------------------------------------
void ofApp::keyPressed(int key){
//std::cout << "key: " << key << std::endl;
switch(key) {
case 32:
recordingState = 1;
break;
case 13:
modelControl = (modelControl) ? false : true;
break;
}
}
//--------------------------------------------------------------
void ofApp::keyReleased(int key){
recordingState = 0;
if (trainingSet.size() > 0) {
trained = myNN.train(trainingSet);
std::cout << "trained: " << trained << std::endl;
}
}
//--------------------------------------------------------------
void ofApp::mouseMoved(int x, int y ){
if (inputDevice == false) { //don't do this with Myo is on
if (recordingState > 0) {
trainingExample tempExample;
tempExample.input = { double(x), double(y) };
for (int i = 0; i < allSliders.size() ; ++i) {
tempExample.output.push_back(double(allSliders[i]));
}
trainingSet.push_back(tempExample);
} else if (trained && modelControl == 1) {
std::vector<double> inputVec;
inputVec.push_back(double(x));
inputVec.push_back(double(y));
std::vector<double> output = myNN.run(inputVec);
for (int i = 0; i < output.size(); ++i) {
allSliders[i] = int(output[i]);
}
}
}
}
//--------------------------------------------------------------
void ofApp::mouseDragged(int x, int y, int button){
}
//--------------------------------------------------------------
void ofApp::mousePressed(int x, int y, int button){
}
//--------------------------------------------------------------
void ofApp::mouseReleased(int x, int y, int button){
}
//--------------------------------------------------------------
void ofApp::mouseEntered(int x, int y){
}
//--------------------------------------------------------------
void ofApp::mouseExited(int x, int y){
}
//--------------------------------------------------------------
void ofApp::windowResized(int w, int h){
}
//--------------------------------------------------------------
void ofApp::gotMessage(ofMessage msg){
}
//--------------------------------------------------------------
void ofApp::dragEvent(ofDragInfo dragInfo){
}
#pragma once
#include <vector>
#include "ofMain.h"
#include "ofxGui.h"
#include "ofxMyo.h"
#include "ofxMaxim.h"
#include "regression.h"
#include "rapidStream.h"
class ofApp : public ofBaseApp{
public:
void setup();
void update();
void draw();
void exit();
void keyPressed(int key);
void keyReleased(int key);
void mouseMoved(int x, int y );
void mouseDragged(int x, int y, int button);
void mousePressed(int x, int y, int button);
void mouseReleased(int x, int y, int button);
void mouseEntered(int x, int y);
void mouseExited(int x, int y);
void windowResized(int w, int h);
void dragEvent(ofDragInfo dragInfo);
void gotMessage(ofMessage msg);
void probsClearPressed();
void resetModelPressed();
ofxIntSlider thung0, thung1, thung2, thung3;
ofxIntSlider thung4, thung5, thung6, thung7;
ofxIntSlider thung8, thung9, thung10, thung11;
std::vector<ofxIntSlider> thungs;
ofxPanel guiThung;
ofxIntSlider tong0, tong1, tong2, tong3;
ofxIntSlider tong4, tong5, tong6, tong7;
ofxIntSlider tong8, tong9, tong10, tong11;
ofxPanel guiTong;
ofxFloatSlider gain;
ofxToggle inputDevice;
ofxToggle modelControl;
ofxButton resetModel;
ofxButton probsClear;
ofxPanel guiGeneral;
std::vector<ofxIntSlider> allSliders;
//Maxi
void audioOut(float * output, int bufferSize, int nChannels);
//void audioIn(float * input, int bufferSize, int nChannels);
int bufferSize;
int initialBufferSize;
int sampleRate;
private:
ofxMyo::Myo myo;
//---------Maxi---------------//
double outputs[2];
maxiSample saron_sbpl1;
maxiSample saron_sbpl2;
maxiSample saron_sbpl3;
maxiSample saron_sbpl4;
maxiSample saron_sbpl5;
maxiSample ciblon_tong;
maxiSample ciblon_thung;
maxiClock myClock;
int currentBeat;
//---------RapidLib---------------//
regression myNN;
std::vector<trainingExample> trainingSet;
int recordingState;
bool trained;
rapidStream streamBuf = rapidStream(25);
};
#!/bin/bash
#Make sure we've got the latest version
git pull
git submodule update --init --recursive
rm -rf build/
mkdir build
cd build
cmake ..
make
./rapidmixTest
cp ./helloRapidMix ../examples/HelloRapidMix/helloRapidMix
\ No newline at end of file
Copyright (c) 2017 Goldsmiths College University of London
Copyright (c) 2017 by IRCAM – Centre Pompidou, Paris, France
All rights reserved.
The RAPID-MIX API wrapper, in the /src directory, is licenced by the BSD license below. Submodules in the /dependances
folder have their own copyrights and licenses, including MIT, BSD, and GPLv3 licenses. Users are requested to check
individual folders for license details, or to contact RAPID-MIX developers.
BSD 3-clause
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
following conditions are met:
- Redistributions of source code must retain the above copyright notice, this list of conditions and the following
disclaimer.
- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided with the distribution.
- Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
/*
* machineLearning.cpp
* Created by Michael Zbyszynski on 10 Jan 2016
/**
* @file machineLearning.cpp
* @author Michael Zbyszynski
* @date 10 Jan 2016
* @copyright
* Copyright © 2017 Goldsmiths. All rights reserved.
*/
#include "machineLearning.h"
RAPIDMIX_BEGIN_NAMESPACE
void trainingData2rapidLib (const trainingData &newTrainingData, std::vector<trainingExample> &trainingSet) {
for (int h = 0; h < newTrainingData.trainingSet.size(); ++h) { //Go through every phrase
for (int i = 0; i < newTrainingData.trainingSet[h].elements.size(); ++i) { //...and every element
trainingExample tempExample;
tempExample.input = newTrainingData.trainingSet[h].elements[i].input;
if (newTrainingData.trainingSet[h].elements[i].output.size() > 0) {
tempExample.output = newTrainingData.trainingSet[h].elements[i].output;
} else {
std::unordered_map<std::string, int>::const_iterator mappedLabel = newTrainingData.labels.find(newTrainingData.trainingSet[h].label);
tempExample.output.push_back(double(mappedLabel->second));
namespace rapidmix {
//////////////////////////////////////////////////////////////////////// Generic train
template <class MachineLearningModule>
bool machineLearning<MachineLearningModule>::train(const trainingData &newTrainingData) {
return MachineLearningModule::train(newTrainingData);
}
/////////////////////////////////////////////////////////////////////// RapidLib specializations
void trainingData2rapidLib (const trainingData &newTrainingData, std::vector<trainingExample> &trainingSet) {
for (int h = 0; h < newTrainingData.trainingSet.size(); ++h) { //Go through every phrase
for (int i = 0; i < newTrainingData.trainingSet[h].elements.size(); ++i) { //...and every element
trainingExample tempExample;
tempExample.input = newTrainingData.trainingSet[h].elements[i].input;
if (newTrainingData.trainingSet[h].elements[i].output.size() > 0) {
tempExample.output = newTrainingData.trainingSet[h].elements[i].output;
} else {
tempExample.output.push_back(double(h));
}
trainingSet.push_back(tempExample);
}
trainingSet.push_back(tempExample);
}
};
/////////////////////////////////////////////////////////////////////// RapidLib classification
template<>
bool machineLearning<classification>::train(const trainingData &newTrainingData) {
std::vector<trainingExample> trainingSet;
labels.clear();
for (int i = 0; i < newTrainingData.trainingSet.size(); ++i) {
labels.push_back(newTrainingData.trainingSet[i].label);
}
trainingData2rapidLib(newTrainingData, trainingSet);
return classification::train(trainingSet);
}
template<>
std::string machineLearning<classification>::run(const std::vector<double> &inputVector, const std::string &label) {
int classIndex = classification::run(inputVector)[0];
return labels[classIndex];
};
/////////////////////////////////////////////////////////////////////// RapidLib regression
template<>
bool machineLearning<regression>::train(const trainingData &newTrainingData) {
std::vector<trainingExample> trainingSet;
trainingData2rapidLib(newTrainingData, trainingSet);
return regression::train(trainingSet);
}
/////////////////////////////////////////////////////////////////////// RapidLib seriesClassification
template<>
bool machineLearning<seriesClassification>::train(const trainingData &newTrainingData) {
std::vector<trainingSeries> seriesSet;
for (int i = 0; i < newTrainingData.trainingSet.size(); ++i) { //each phrase
trainingSeries tempSeries;
tempSeries.label = newTrainingData.trainingSet[i].label;
for (int j = 0; j < newTrainingData.trainingSet[i].elements.size(); ++j) { //each element
tempSeries.input.push_back(newTrainingData.trainingSet[i].elements[j].input);
}
seriesSet.push_back(tempSeries);
}
return seriesClassification::train(seriesSet);
}
template<>
std::string machineLearning<seriesClassification>::run(const std::vector<std::vector<double> > &inputSeries) {
return seriesClassification::run(inputSeries);
}
/////////////////////////////////////////////////////////////////////// GVF
template<>
bool machineLearning<rapidGVF>::train(const trainingData &newTrainingData) {
return rapidGVF::train(newTrainingData);
}
};
template<>
bool machineLearning<classification>::train(const trainingData &newTrainingData) {
std::vector<trainingExample> trainingSet;
trainingData2rapidLib(newTrainingData, trainingSet);
return classification::train(trainingSet);
}
template<>
bool machineLearning<regression>::train(const trainingData &newTrainingData) {
std::vector<trainingExample> trainingSet;
trainingData2rapidLib(newTrainingData, trainingSet);
return regression::train(trainingSet);
}
RAPIDMIX_END_NAMESPACE
/*
* machineLearning.h
* Created by Michael Zbyszynski on 10 Jan 2016
/**
* @file machineLearning.h
* @author Michael Zbyszynski on 10 Jan 2016
* @copyright
* Copyright © 2017 Goldsmiths. All rights reserved.
*
* @ingroup machinelearning
*/
#ifndef machineLearning_h
#define machineLearning_h
#include "rapidMix.h"
#include "trainingData.h"
#include <vector>
#include "../rapidmix.h"
////////// Include all of the machine learning algorithms here
#include "classification.h"
#include "regression.h"
//#include "rapidXmmTools.h"
//#include "gvf.h"
#include "seriesClassification.h"
#include "./rapidXMM/rapidXMM.h"
#include "./rapidGVF/rapidGVF.h"
// forward declaration
namespace rapidmix { class trainingData; }
RAPIDMIX_BEGIN_NAMESPACE
//* Host class for machine learning algorithms */
template <typename MachineLearningModule>
class machineLearning : public MachineLearningModule {
public:
namespace rapidmix {
//* Constructors */
machineLearning() : MachineLearningModule() {};
// forward declaration
class trainingData;
template<class T>
machineLearning(T type) : MachineLearningModule(type) {};
/** @brief A generic ouptut struct to fit all kinds of models */
typedef struct runResults_t {
std::vector<double> likelihoods;
std::vector<double> regression;
std::vector<double> progressions;
std::string likeliest;
} runResults;
//* this function becomes specialized in the implementation */
bool train(const trainingData &newTrainingData);
/**
* @brief Host class for machine learning algorithms
*/
template <typename MachineLearningModule>
class machineLearning : public MachineLearningModule {
public:
//* Constructors */
machineLearning() : MachineLearningModule() {};
template<class T>
machineLearning(T type) : MachineLearningModule(type) {};
/**
* @brief This function becomes specialized in the implementation
*/
bool train(const trainingData &newTrainingData);
//* this function is not being specialized
std::vector<double> run(const std::vector<double> &inputVector) {
return MachineLearningModule::run(inputVector);
}
// This is a hack while I think about how to do this. -MZ //
std::string run(const std::vector<double> &inputVector, const std::string &label);
//* This is the one I'm using for DTW */
std::string run(const std::vector<std::vector<double> > &inputSeries);
bool reset() {
return MachineLearningModule::reset();
}
private:
MachineLearningModule module;
//this holds string labels
std::vector<std::string> labels; //FIXME: This probably should be pushed down into rapidLib?
std::string getLabel(int value);
};
// Could overload this, or specialize, or both
std::vector<double> run(const std::vector<double> &inputVector) {
return MachineLearningModule::process(inputVector);
}
////////// typedefs for calling different algorithms
bool reset() {
return MachineLearningModule::reset();
}
///// RapidLib
private:
MachineLearningModule module;
};
////////// typedefs for calling different algorithms
typedef machineLearning<classification> staticClassification;
typedef machineLearning<regression> staticRegression;
/*
typedef xmmToolConfig xmmConfig;
typedef machineLearning<xmmGmmTool> xmmStaticClassification;
typedef machineLearning<xmmGmrTool> xmmStaticRegression;
typedef machineLearning<xmmHmmTool> xmmTemporalClassification;
typedef machineLearning<xmmHmrTool> xmmTemporalRegression;
typedef machineLearning<GVF> gvf;
*/
RAPIDMIX_END_NAMESPACE
/** @brief static classification using KNN from RapidLib */
typedef machineLearning<classification> staticClassification;
/** @brief static regression using Neural Networks from RapidLib */
typedef machineLearning<regression> staticRegression;
/** @brief temporal classification using Dynamic Time Warping from RapidLib */
typedef machineLearning<seriesClassification> dtwTemporalClassification;
///// XMM
/** @brief configuration for XMM based algorithms */
typedef xmmToolConfig xmmConfig;
/** @brief static classification using Gaussian Mixture Models from XMM */
typedef machineLearning<rapidXmmGmm> xmmStaticClassification;
/** @brief static regression using Gaussian Mixture Models from XMM */
typedef machineLearning<rapidXmmGmr> xmmStaticRegression;
/** @brief temporal classification using Hierarchical Hidden Markov Models from XMM */
typedef machineLearning<rapidXmmHmm> xmmTemporalClassification;
/** @brief temporal regression using Hierarchical Hidden Markov Models from XMM */
typedef machineLearning<rapidXmmHmr> xmmTemporalRegression;
///// GVF
/** @brief temporal variation estimation using GVF library */
typedef machineLearning<rapidGVF> gvfTemporalVariation;
}
#endif
//
// rapidGVF.cpp
//
// Created by Francisco on 04/05/2017.
// Copyright © 2017 Goldsmiths. All rights reserved.
//
#include "rapidGVF.h"
#include "../trainingData.h"
rapidGVF::rapidGVF() {}
rapidGVF::~rapidGVF() {}
bool rapidGVF::train(const rapidmix::trainingData &newTrainingData)
{
if (newTrainingData.trainingSet.size() < 1)
{
// no recorded phrase
return false;
}
if (newTrainingData.trainingSet.size() == 1 && newTrainingData.trainingSet[0].elements.size() == 0) {
// empty recorded phrase
return false;
}
if(gvf.getState() != GVF::STATE_LEARNING)
{
gvf.setState(GVF::STATE_LEARNING);
}
//Go through every phrase
for (int h = 0; h < newTrainingData.trainingSet.size(); ++h)
{
gvf.startGesture();
for (int i = 0; i < newTrainingData.trainingSet[h].elements.size(); ++i)
{
std::vector<double> vd = newTrainingData.trainingSet[h].elements[i].input;
// Using template <class InputIterator> vector to change for vec<double> to vec<float>
std::vector<float> vf(vd.begin(), vd.end());
this->currentGesture.addObservation(vf);
}
gvf.addGestureTemplate(this->currentGesture);
}
return true;
}
std::vector<double> rapidGVF::run(const std::vector<double> &inputVector)
{
if (inputVector.size() == 0)
{
return std::vector<double>();
}
gvf.restart();
if (gvf.getState() != GVF::STATE_FOLLOWING)
{
gvf.setState(GVF::STATE_FOLLOWING);
}
// Using template <class InputIterator> vector to change for vec<double> to vec<float>
std::vector<float> vf(inputVector.begin(),inputVector.end());
this->currentGesture.addObservation(vf);
outcomes = gvf.update(this->currentGesture.getLastObservation());
std::vector<double> output;
output.push_back(outcomes.likeliestGesture);
output.insert(output.end(), outcomes.likelihoods.begin(), outcomes.likelihoods.end());
output.insert(output.end(), outcomes.alignments.begin(), outcomes.alignments.end());
return output;
}
const std::vector<float> rapidGVF::getLikelihoods()
{
return outcomes.likelihoods;
};
const std::vector<float> rapidGVF::getAlignments()
{
return outcomes.alignments;
};
const std::vector<std::vector<float> > * rapidGVF::getDynamics()
{
return &outcomes.dynamics;
};
const std::vector<std::vector<float> > * rapidGVF::getScalings()
{
return &outcomes.scalings;
};
const std::vector<std::vector<float> > * rapidGVF::getRotations()
{
return &outcomes.rotations;
};
/**
* @file rapidGVF.h
* Created by Francisco on 04/05/2017.
* Copyright © 2017 Goldsmiths. All rights reserved.
*
* @ingroup machinelearning
*/
#ifndef rapidGVF_h
#define rapidGVF_h
#include <vector>
#include <string>
#include "GVF.h"
namespace rapidmix { class trainingData; }
/**
* @brief This class is an adapter for the GVF library from Baptiste Caramiaux
*/
class rapidGVF {
public:
rapidGVF();
~rapidGVF();
bool train(const rapidmix::trainingData &newTrainingData);
std::vector<double> run(const std::vector<double> &inputVector);
//TODO: needs a "reset" message
const std::vector<float> getLikelihoods();
const std::vector<float> getAlignments();
const std::vector<std::vector<float> > * getDynamics();
const std::vector<std::vector<float> > * getScalings();
const std::vector<std::vector<float> > * getRotations();
private:
GVF gvf;
GVFGesture currentGesture;
GVFOutcomes outcomes;
};
#endif
#include "rapidXMM.h"
#include "../trainingData.h"
#include "../machineLearning.h"
static bool trainingData2xmmTrainingSet(const rapidmix::trainingData& data, xmm::TrainingSet& set) {
if (data.trainingSet.size() < 1) {
return false;
}
if (data.trainingSet.size() > 0 && data.trainingSet[0].elements.size() == 0) {
// empty recorded phrase
return false;
}
rapidmix::trainingData::element el = data.trainingSet[0].elements[0];
int dimIn = static_cast<int>(el.input.size());
int dimOut = static_cast<int>(el.output.size());
// translate and return true if data and set are compatible
// don't translate and return false otherwise
if (dimOut > 0 != set.bimodal()) {
return false;
}
xmm::Phrase xp;
if (set.bimodal()) {
set.dimension.set(dimIn + dimOut);
set.dimension_input.set(dimIn);
xp = xmm::Phrase(xmm::MemoryMode::OwnMemory, xmm::Multimodality::Bimodal);
xp.dimension.set(dimIn + dimOut);
xp.dimension_input.set(dimIn);
} else {
set.dimension.set(dimIn);
set.dimension_input.set(0);
xp = xmm::Phrase(xmm::MemoryMode::OwnMemory, xmm::Multimodality::Unimodal);
xp.dimension.set(dimIn);
xp.dimension_input.set(0);
}
set.clear();
//for (auto &phrase : data.trainingSet) {
for (int i = 0; i < data.trainingSet.size(); ++i) {
const rapidmix::trainingData::phrase &phrase = data.trainingSet[i];
xp.clear();
xp.label.set(phrase.label);
for (auto &element : phrase.elements) {
std::vector<float> obsIn(element.input.begin(), element.input.end());
std::vector<float> obsOut(element.output.begin(), element.output.end());
std::vector<float> obs;
obs.insert(obs.end(), obsIn.begin(), obsIn.end());
obs.insert(obs.end(), obsOut.begin(), obsOut.end());
xp.record(obs);
}
set.addPhrase(static_cast<int>(set.size()), xp);
}
return true;
}
//=============================== xmmTool ====================================//
template <class SingleClassModel, class Model>
bool xmmTool<SingleClassModel, Model>::train(const rapidmix::trainingData& newTrainingData) {
if (trainingData2xmmTrainingSet(newTrainingData, set)) {
model.train(&set);
model.reset();
return true;
}
return false;
}
////////// private JSON data manipulation methods :
//TODO: add a type field (gmm/gmr/hmm/hmr) in metadata when family is xmm
template <class SingleClassModel, class Model>
Json::Value xmmTool<SingleClassModel, Model>::toJSON(/*std::string modelType*/) {
Json::Value root;
root["docType"] = "rapid-mix:ml-model";
root["docVersion"] = RAPIDMIX_JSON_DOC_VERSION;
Json::Value target;
target["name"] = "xmm";
target["version"] = "v1.0.0";
root["target"] = target;
root["payload"] = model.toJson();
return root;
}
template <class SingleClassModel, class Model>
bool xmmTool<SingleClassModel, Model>::fromJSON(Json::Value &jm) {
if (jm["docType"].asString().compare("rapid-mix:ml-model") == 0 &&
jm["target"]["name"].asString().compare("xmm") == 0 &&
jm["payload"].size() > 0) {
model.fromJson(jm["payload"]);
model.reset();
return true;
}
return false;
}
////////// public JSON file manipulation interface :
template <class SingleClassModel, class Model>
std::string xmmTool<SingleClassModel, Model>::getJSON() {
Json::Value result = toJSON();
return result.toStyledString();
}
template <class SingleClassModel, class Model>
void xmmTool<SingleClassModel, Model>::writeJSON(const std::string &filepath) {
Json::Value root = toJSON();
std::ofstream jsonOut;
jsonOut.open (filepath);
Json::StyledStreamWriter writer;
writer.write(jsonOut, root);
jsonOut.close();
}
template <class SingleClassModel, class Model>
bool xmmTool<SingleClassModel, Model>::putJSON(const std::string &jsonMessage) {
Json::Value parsedFromString;
Json::Reader reader;
bool parsingSuccessful = reader.parse(jsonMessage, parsedFromString);
return (parsingSuccessful && fromJSON(parsedFromString));
}
template <class SingleClassModel, class Model>
bool xmmTool<SingleClassModel, Model>::readJSON(const std::string &filepath) {
Json::Value root;
std::ifstream file(filepath);
file >> root;
return fromJSON(root);
}
//============================== xmmGmmTool ==================================//
std::vector<double> rapidXmmGmm::run(const std::vector<double>& inputVector) {
xmmTool::preProcess(inputVector);
return model.results.smoothed_normalized_likelihoods;
}
//============================== xmmGmrTool ==================================//
std::vector<double> rapidXmmGmr::run(const std::vector<double>& inputVector) {
xmmTool::preProcess(inputVector);
std::vector<float> *res = &model.results.output_values;
std::vector<double> dRes(res->begin(), res->end());
return dRes;
}
//============================== xmmHmmTool ==================================//
std::vector<double> rapidXmmHmm::run(const std::vector<double>& inputVector) {
xmmTool::preProcess(inputVector);
std::vector<double> res;
int i(0);
for (auto &m : model.models) {
res.push_back(model.results.smoothed_normalized_likelihoods[i]);
res.push_back(m.second.results.progress);
i++;
}
return res;
}
//============================== xmmHmrTool ==================================//
std::vector<double> rapidXmmHmr::run(const std::vector<double>& inputVector) {
xmmTool::preProcess(inputVector);
std::vector<float> *res = &model.results.output_values;
std::vector<double> dRes(res->begin(), res->end());
return dRes;
}
///////////////////////////////////////////////////////////////////////////
///// generic train method and forward declaration of specialized templates
///////////////////////////////////////////////////////////////////////////
template class xmmTool<xmm::GMM, xmm::GMM>;
template class xmmTool<xmm::HMM, xmm::HierarchicalHMM>;
template class xmmStaticTool<xmm::GMM, xmm::GMM>;
template class xmmTemporalTool<xmm::HMM, xmm::HierarchicalHMM>;
//I wonder why this can't be defined in machineLearning.cpp? -MZ
// It is needed by the template instantiations below.
// You get an undefined symbols error otherwise.
template <class MachineLearningModule>
bool rapidmix::machineLearning<MachineLearningModule>::train(const trainingData &newTrainingData) {
return MachineLearningModule::train(newTrainingData);
}
template class rapidmix::machineLearning<rapidXmmGmm>;
template class rapidmix::machineLearning<rapidXmmGmr>;
template class rapidmix::machineLearning<rapidXmmHmm>;
template class rapidmix::machineLearning<rapidXmmHmr>;
/**
* @file rapidXMM.h
* @author joseph larralde
*
* @copyright
* Copyright (C) 2016 - 2017 by IRCAM - Centre Pompidou, Paris, France.
* All rights reserved.
*
* @ingroup machinelearning
*/
#ifndef _RAPID_XMM_TOOLS_H_
#define _RAPID_XMM_TOOLS_H_
// this works !
#ifndef EXTERNAL_JSONCPP_PATH
// #define EXTERNAL_JSONCPP_PATH "../../../../json/json.h" // relative to xmmJson.h
#define EXTERNAL_JSONCPP_PATH "json.h"
#endif /* EXTERNAL_JSONCPP_PATH */
#include "xmm.h"
// forward declaration
namespace rapidmix { class trainingData; }
// original defined in xmmModelConfiguration.hpp
enum xmmRegressionEstimator {
xmmLikeliestRegression,
xmmMixtureRegression
};
// original defined in xmmGaussianDistribution.hpp
enum xmmCovarianceMode {
xmmFullCovariance,
xmmDiagonalCovariance
};
// original defined in xmmHmmParameters.hpp
enum xmmHmmTransitionMode {
xmmHmmLeftRightTransition,
xmmHmmErgodicTransition
};
// original defined in xmmHmmParameters.hpp
enum xmmHmmRegressionEstimator {
xmmHmmFullRegression,
xmmHmmWindowedRegression,
xmmHmmLikeliestRegression
};
// this is the optional argument of machineLearning<xmmWhateverTool>'s constructors
struct xmmToolConfig {
xmmToolConfig() :
gaussians(1),
relativeRegularization(0.01),
absoluteRegularization(0.01),
regressionEstimator(xmmMixtureRegression),
covarianceMode(xmmFullCovariance),
states(5),
hierarchical(true),
hmmTransitionMode(xmmHmmErgodicTransition),
hmmRegressionEstimator(xmmHmmFullRegression),
likelihoodWindow(5) {}
// general parameters :
uint32_t gaussians;
float relativeRegularization;
float absoluteRegularization;
xmmRegressionEstimator regressionEstimator;
xmmCovarianceMode covarianceMode;
// hmm specific :
uint32_t states;
bool hierarchical;
xmmHmmTransitionMode hmmTransitionMode;
xmmHmmRegressionEstimator hmmRegressionEstimator;
// run-time parameter :
uint32_t likelihoodWindow;
};
//========================== template base class =============================//
template <class SingleClassModel, class Model>
class xmmTool {
protected:
xmmTool(bool bimodal) {
model = Model(bimodal);
model.configuration.multithreading = xmm::MultithreadingMode::Sequential;
model.configuration.changed = true;
set = xmm::TrainingSet(xmm::MemoryMode::OwnMemory,
bimodal
? xmm::Multimodality::Bimodal
: xmm::Multimodality::Unimodal
);
}
virtual void preProcess(const std::vector<double> &inputVector) {
std::vector<float> fv(inputVector.begin(), inputVector.end());
model.filter(fv);
}
public:
virtual ~xmmTool() {}
virtual bool train(const rapidmix::trainingData &newTrainingData);
virtual bool reset() {
model.reset();
return true;
}
/** Get a JSON representation of the model in the form of a styled string */
virtual std::string getJSON();
/** Write a JSON model description to specified file path */
virtual void writeJSON(const std::string &filepath);
/** configure empty model with string. See getJSON() */
virtual bool putJSON(const std::string &jsonMessage);
/** read a JSON file at file path and build a modelSet from it */
virtual bool readJSON(const std::string &filepath);
protected:
Model model;
xmm::TrainingSet set;
Json::Value toJSON();
bool fromJSON(Json::Value &jm);
};
//======================= base class for GMM models ==========================//
template <class SingleClassModel, class Model>
class xmmStaticTool : public xmmTool<SingleClassModel, Model> {
protected:
xmmStaticTool(xmmToolConfig cfg, bool bimodal) :
xmmTool<SingleClassModel, Model>(bimodal) {
xmm::Configuration<SingleClassModel>& mCfg = this->model.configuration;
mCfg.gaussians.set(cfg.gaussians);
mCfg.relative_regularization.set(cfg.relativeRegularization);
mCfg.absolute_regularization.set(cfg.absoluteRegularization);
xmm::MultiClassRegressionEstimator mcre;
switch (cfg.regressionEstimator) {
case xmmLikeliestRegression:
mcre = xmm::MultiClassRegressionEstimator::Likeliest;
case xmmMixtureRegression:
default:
mcre = xmm::MultiClassRegressionEstimator::Mixture;
break;
}
mCfg.multiClass_regression_estimator = mcre;
xmm::GaussianDistribution::CovarianceMode gdcm;
switch (cfg.covarianceMode) {
case xmmFullCovariance:
gdcm = xmm::GaussianDistribution::CovarianceMode::Full;
break;
case xmmDiagonalCovariance:
default:
gdcm = xmm::GaussianDistribution::CovarianceMode::Diagonal;
break;
}
mCfg.covariance_mode.set(gdcm);
mCfg.changed = true;
this->model.shared_parameters->likelihood_window.set(cfg.likelihoodWindow);
}
public:
virtual ~xmmStaticTool() {}
};
//======================= base class for HMM models ==========================//
template <class SingleClassModel, class Model>
class xmmTemporalTool : public xmmStaticTool<SingleClassModel, Model> {
protected:
xmmTemporalTool(xmmToolConfig cfg, bool bimodal) :
xmmStaticTool<SingleClassModel, Model>(cfg, bimodal) {
xmm::Configuration<SingleClassModel>& mCfg = this->model.configuration;
mCfg.states.set(cfg.states);
mCfg.hierarchical.set(cfg.hierarchical);
xmm::HMM::TransitionMode htm;
switch (cfg.hmmTransitionMode) {
case xmmHmmLeftRightTransition:
htm = xmm::HMM::TransitionMode::LeftRight;
break;
case xmmHmmErgodicTransition:
default:
htm = xmm::HMM::TransitionMode::Ergodic;
break;
}
mCfg.transition_mode.set(htm);
xmm::HMM::RegressionEstimator hre;
switch (cfg.hmmRegressionEstimator) {
case xmmHmmFullRegression:
hre = xmm::HMM::RegressionEstimator::Full;
break;
case xmmHmmWindowedRegression:
hre = xmm::HMM::RegressionEstimator::Windowed;
break;
case xmmHmmLikeliestRegression:
default:
hre = xmm::HMM::RegressionEstimator::Likeliest;
break;
}
mCfg.regression_estimator.set(hre);
mCfg.changed = true;
}
public:
virtual ~xmmTemporalTool() {}
};
//================== actual classes used in machineLearning.h ================//
/**
* @brief Static classification using Gaussian Mixture Models
*/
class rapidXmmGmm : public xmmStaticTool<xmm::GMM, xmm::GMM> {
public:
rapidXmmGmm(xmmToolConfig cfg = xmmToolConfig()) :
xmmStaticTool<xmm::GMM, xmm::GMM>(cfg, false) {}
~rapidXmmGmm() {}
std::vector<double> run(const std::vector<double>& inputVector);
};
/**
* @brief Static regression using Gaussian Mixture Models
*/
class rapidXmmGmr : public xmmStaticTool<xmm::GMM, xmm::GMM> {
public:
rapidXmmGmr(xmmToolConfig cfg = xmmToolConfig()) :
xmmStaticTool<xmm::GMM, xmm::GMM>(cfg, true) {}
~rapidXmmGmr() {}
std::vector<double> run(const std::vector<double>& inputVector);
};
/**
* @brief Temporal classification using Hierarchical Hidden Markov Models
*/
class rapidXmmHmm : public xmmTemporalTool<xmm::HMM, xmm::HierarchicalHMM> {
public:
rapidXmmHmm(xmmToolConfig cfg = xmmToolConfig()) :
xmmTemporalTool<xmm::HMM, xmm::HierarchicalHMM>(cfg, false) {}
~rapidXmmHmm() {}
std::vector<double> run(const std::vector<double>& inputVector);
};
/**
* @brief Temporal regression using Hierarchical Hidden Markov Models
*/
class rapidXmmHmr : public xmmTemporalTool<xmm::HMM, xmm::HierarchicalHMM> {
public:
rapidXmmHmr(xmmToolConfig cfg = xmmToolConfig()) :
xmmTemporalTool<xmm::HMM, xmm::HierarchicalHMM>(cfg, true) {}
~rapidXmmHmr() {}
std::vector<double> run(const std::vector<double>& inputVector);
};
#endif /* _RAPID_XMM_TOOLS_H_ */