* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing,

* software distributed under the License is distributed on an

* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

* KIND, either express or implied.  See the License for the

* specific language governing permissions and limitations

* under the License.

*/

    EvaluationOfTemporalRelations_ImplBase {

  /**

   * @param args

   * @throws Exception

   */

  static interface EvalOptions extends Options{

    @Option(longName="iee") boolean getIgnoreEventEvent();

    @Option(longName="iet") boolean getIgnoreEventTime();

  }

 

  private boolean doEventEvent = true;

  private boolean doEventTime = true;

 

  public static void main(String[] args) throws Exception {

    EvalOptions options = CliFactory.parseArguments(EvalOptions.class, args);

    if(options.getIgnoreEventEvent() && options.getIgnoreEventTime()){

      System.err.println("Ignoring all relation types is not a valid configuration.");

      System.exit(-1);

    }

    List<Integer> patientSets = options.getPatients().getList();

    List<Integer> trainItems = THYMEData.getTrainPatientSets(patientSets);

    List<Integer> devItems = THYMEData.getDevPatientSets(patientSets);

    List<Integer> testItems = THYMEData.getTestPatientSets(patientSets);

   

    List<Integer> allTraining = new ArrayList<Integer>(trainItems);

    List<Integer> allTest;

    if (options.getTest()) {

      allTraining.addAll(devItems);

      allTest = new ArrayList<Integer>(testItems);

    } else {

      allTest = new ArrayList<Integer>(devItems);

    }

   

    EvaluationOfClearTKRelations evaluation = new EvaluationOfClearTKRelations(

        new File("target/eval/cleartk-event-time-links"),

        options.getRawTextDirectory(),

        options.getXMLDirectory(),

        options.getXMLFormat(),

        options.getXMIDirectory());

    evaluation.setExtractEventEvent(!options.getIgnoreEventEvent());

    evaluation.setExtractEventTime(!options.getIgnoreEventTime());

   

    evaluation.prepareXMIsFor(patientSets);

    AnnotationStatistics<String> stats = evaluation.trainAndTest(allTraining, allTest);

    System.err.println(stats);

  }

 

 

 

  public EvaluationOfClearTKRelations(File baseDirectory, File rawTextDirectory,

      File xmlDirectory,

      org.apache.ctakes.temporal.eval.Evaluation_ImplBase.XMLFormat xmlFormat,

      File xmiDirectory) {

    super(baseDirectory, rawTextDirectory, xmlDirectory, xmlFormat, xmiDirectory, null, false, false, defaultParams);

  }

 

  private void setExtractEventTime(boolean eventTime) {

    this.doEventTime = eventTime;

  }

  private void setExtractEventEvent(boolean eventEvent) {

    this.doEventEvent = eventEvent;

  }

  @Override

  protected void train(CollectionReader collectionReader, File directory)

      throws Exception {

    // not training a model - just using the ClearTK one

  }

 

  @Override

  protected AnnotationStatistics<String> test(

      CollectionReader collectionReader, File directory) throws Exception {

    AggregateBuilder aggregateBuilder = this.getPreprocessorAggregateBuilder();

    aggregateBuilder.add(CopyFromGold.getDescription(EventMention.class, TimeMention.class));

    aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(

        RemoveCrossSentenceRelations.class,

        RemoveCrossSentenceRelations.PARAM_SENTENCE_VIEW,

        CAS.NAME_DEFAULT_SOFA,

        RemoveCrossSentenceRelations.PARAM_RELATION_VIEW,

        GOLD_VIEW_NAME));

    if(!this.doEventEvent){

      aggregateBuilder.add(

          AnalysisEngineFactory.createPrimitiveDescription(RemoveEventEventRelations.class),

          CAS.NAME_DEFAULT_SOFA,

          GOLD_VIEW_NAME);

    }

    if(!this.doEventTime){

      aggregateBuilder.add(

          AnalysisEngineFactory.createPrimitiveDescription(PreserveEventEventRelations.class),

          CAS.NAME_DEFAULT_SOFA,

          GOLD_VIEW_NAME);

    }

    aggregateBuilder.add(SentenceAnnotator.getDescription());

    aggregateBuilder.add(TokenAnnotator.getDescription());

    aggregateBuilder.add(PosTaggerAnnotator.getDescription());

    aggregateBuilder.add(DefaultSnowballStemmer.getDescription("English"));

    aggregateBuilder.add(ParserAnnotator.getDescription());

    aggregateBuilder.add(EventToClearTKEventAnnotator.getAnnotatorDescription());//for every cTakes eventMention, create a cleartk event

    aggregateBuilder.add(TimexToClearTKTimexAnnotator.getAnnotatorDescription());

    aggregateBuilder.add(EventTenseAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/event/eventtenseannotator/model.jar"));

    aggregateBuilder.add(EventAspectAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/event/eventaspectannotator/model.jar"));

    aggregateBuilder.add(EventClassAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/event/eventclassannotator/model.jar"));

    aggregateBuilder.add(EventPolarityAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/event/eventpolarityannotator/model.jar"));

    aggregateBuilder.add(EventModalityAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/event/eventmodalityannotator/model.jar"));

    aggregateBuilder.add(TimeTypeAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/time/timetypeannotator/model.jar"));

//    aggregateBuilder.add(ClearTKDocTimeRelAnnotator.getAnnotatorDescription());// for every tlink, check if it cover and event, add the tlink type to the event's docTimeRel attribute

    if(this.doEventTime){

      aggregateBuilder.add(TemporalLinkEventToSameSentenceTimeAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/tlink/temporallinkeventtosamesentencetimeannotator/model.jar"));

    }

    if(this.doEventEvent){

      aggregateBuilder.add(TemporalLinkEventToSubordinatedEventAnnotator.FACTORY.getAnnotatorDescription("/org/cleartk/timeml/tlink/temporallinkeventtosubordinatedeventannotator/model.jar"));

    }

   

    aggregateBuilder.add(ClearTKLinkToTHYMELinkAnnotator.getAnnotatorDescription());

    Function<BinaryTextRelation, ?> getSpan = new Function<BinaryTextRelation, HashableArguments>() {

      public HashableArguments apply(BinaryTextRelation relation) {

        return new HashableArguments(relation);

      }

    };

    Function<BinaryTextRelation, String> getOutcome = AnnotationStatistics.annotationToFeatureValue("category");

    AnnotationStatistics<String> stats = new AnnotationStatistics<String>();

    for (JCas jCas : new JCasIterable(collectionReader, aggregateBuilder.createAggregate())) {

      JCas goldView = jCas.getView(GOLD_VIEW_NAME);

      JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA);

      Collection<BinaryTextRelation> goldRelations = JCasUtil.select(

          goldView,

          BinaryTextRelation.class);

      Collection<BinaryTextRelation> systemRelations = JCasUtil.select(

          systemView,

          BinaryTextRelation.class);

      stats.add(goldRelations, systemRelations, getSpan, getOutcome);

    }

   

    return stats;

  }

}