MOA 12.03
Real Time Analytics for Data Streams
EvaluateModel.java
Go to the documentation of this file.
00001 /*
00002  *    EvaluateModel.java
00003  *    Copyright (C) 2007 University of Waikato, Hamilton, New Zealand
00004  *    @author Richard Kirkby (rkirkby@cs.waikato.ac.nz)
00005  *
00006  *    This program is free software; you can redistribute it and/or modify
00007  *    it under the terms of the GNU General Public License as published by
00008  *    the Free Software Foundation; either version 3 of the License, or
00009  *    (at your option) any later version.
00010  *
00011  *    This program is distributed in the hope that it will be useful,
00012  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  *    GNU General Public License for more details.
00015  *
00016  *    You should have received a copy of the GNU General Public License
00017  *    along with this program. If not, see <http://www.gnu.org/licenses/>.
00018  *    
00019  */
00020 package moa.tasks;
00021 
00022 import java.io.File;
00023 import java.io.FileOutputStream;
00024 import java.io.PrintStream;
00025 import moa.classifiers.Classifier;
00026 import moa.core.ObjectRepository;
00027 import moa.evaluation.ClassificationPerformanceEvaluator;
00028 import moa.evaluation.LearningEvaluation;
00029 import moa.options.ClassOption;
00030 import moa.options.FileOption;
00031 import moa.options.IntOption;
00032 import moa.streams.InstanceStream;
00033 import weka.core.Instance;
00034 import weka.core.Utils;
00035 
00042 public class EvaluateModel extends MainTask {
00043 
00044     @Override
00045     public String getPurposeString() {
00046         return "Evaluates a static model on a stream.";
00047     }
00048 
00049     private static final long serialVersionUID = 1L;
00050 
00051     public ClassOption modelOption = new ClassOption("model", 'm',
00052             "Classifier to evaluate.", Classifier.class, "LearnModel");
00053 
00054     public ClassOption streamOption = new ClassOption("stream", 's',
00055             "Stream to evaluate on.", InstanceStream.class,
00056             "generators.RandomTreeGenerator");
00057 
00058     public ClassOption evaluatorOption = new ClassOption("evaluator", 'e',
00059             "Classification performance evaluation method.",
00060             ClassificationPerformanceEvaluator.class,
00061             "BasicClassificationPerformanceEvaluator");
00062 
00063     public IntOption maxInstancesOption = new IntOption("maxInstances", 'i',
00064             "Maximum number of instances to test.", 1000000, 0,
00065             Integer.MAX_VALUE);
00066 
00067     public FileOption outputPredictionFileOption = new FileOption("outputPredictionFile", 'o',
00068             "File to append output predictions to.", null, "pred", true);
00069 
00070     public EvaluateModel() {
00071     }
00072 
00073     public EvaluateModel(Classifier model, InstanceStream stream,
00074             ClassificationPerformanceEvaluator evaluator, int maxInstances) {
00075         this.modelOption.setCurrentObject(model);
00076         this.streamOption.setCurrentObject(stream);
00077         this.evaluatorOption.setCurrentObject(evaluator);
00078         this.maxInstancesOption.setValue(maxInstances);
00079     }
00080 
00081     @Override
00082     public Class<?> getTaskResultType() {
00083         return LearningEvaluation.class;
00084     }
00085 
00086     @Override
00087     public Object doMainTask(TaskMonitor monitor, ObjectRepository repository) {
00088         Classifier model = (Classifier) getPreparedClassOption(this.modelOption);
00089         InstanceStream stream = (InstanceStream) getPreparedClassOption(this.streamOption);
00090         ClassificationPerformanceEvaluator evaluator = (ClassificationPerformanceEvaluator) getPreparedClassOption(this.evaluatorOption);
00091         int maxInstances = this.maxInstancesOption.getValue();
00092         long instancesProcessed = 0;
00093         monitor.setCurrentActivity("Evaluating model...", -1.0);
00094 
00095         //File for output predictions
00096         File outputPredictionFile = this.outputPredictionFileOption.getFile();
00097         PrintStream outputPredictionResultStream = null;
00098         if (outputPredictionFile != null) {
00099             try {
00100                 if (outputPredictionFile.exists()) {
00101                     outputPredictionResultStream = new PrintStream(
00102                             new FileOutputStream(outputPredictionFile, true), true);
00103                 } else {
00104                     outputPredictionResultStream = new PrintStream(
00105                             new FileOutputStream(outputPredictionFile), true);
00106                 }
00107             } catch (Exception ex) {
00108                 throw new RuntimeException(
00109                         "Unable to open prediction result file: " + outputPredictionFile, ex);
00110             }
00111         }
00112         while (stream.hasMoreInstances()
00113                 && ((maxInstances < 0) || (instancesProcessed < maxInstances))) {
00114             Instance testInst = (Instance) stream.nextInstance().copy();
00115             int trueClass = (int) testInst.classValue();
00116             //testInst.setClassMissing();
00117             double[] prediction = model.getVotesForInstance(testInst);
00118             //evaluator.addClassificationAttempt(trueClass, prediction, testInst
00119             //          .weight());
00120             if (outputPredictionFile != null) {
00121                 outputPredictionResultStream.println(Utils.maxIndex(prediction) + "," + trueClass);
00122             }
00123             evaluator.addResult(testInst, prediction);
00124             instancesProcessed++;
00125             if (instancesProcessed % INSTANCES_BETWEEN_MONITOR_UPDATES == 0) {
00126                 if (monitor.taskShouldAbort()) {
00127                     return null;
00128                 }
00129                 long estimatedRemainingInstances = stream.estimatedRemainingInstances();
00130                 if (maxInstances > 0) {
00131                     long maxRemaining = maxInstances - instancesProcessed;
00132                     if ((estimatedRemainingInstances < 0)
00133                             || (maxRemaining < estimatedRemainingInstances)) {
00134                         estimatedRemainingInstances = maxRemaining;
00135                     }
00136                 }
00137                 monitor.setCurrentActivityFractionComplete(estimatedRemainingInstances < 0 ? -1.0
00138                         : (double) instancesProcessed
00139                         / (double) (instancesProcessed + estimatedRemainingInstances));
00140                 if (monitor.resultPreviewRequested()) {
00141                     monitor.setLatestResultPreview(new LearningEvaluation(
00142                             evaluator.getPerformanceMeasurements()));
00143                 }
00144             }
00145         }
00146         if (outputPredictionResultStream != null) {
00147             outputPredictionResultStream.close();
00148         }
00149         return new LearningEvaluation(evaluator.getPerformanceMeasurements());
00150     }
00151 }
 All Classes Namespaces Files Functions Variables Enumerations