analyzer.h

Go to the documentation of this file.

//
//    FreeLing - Open Source Language Analyzers
//
//    Copyright (C) 2014   TALP Research Center
//                         Universitat Politecnica de Catalunya
//
//    This library is free software; you can redistribute it and/or
//    modify it under the terms of the GNU Affero General Public
//    License as published by the Free Software Foundation; either
//    version 3 of the License, or (at your option) any later version.
//
//    This library is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//    Affero General Public License for more details.
//
//    You should have received a copy of the GNU Affero General Public
//    License along with this library; if not, write to the Free Software
//    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//    contact: Lluis Padro (padro@lsi.upc.es)
//             TALP Research Center
//             despatx C6.212 - Campus Nord UPC
//             08034 Barcelona.  SPAIN
//

#ifndef _ANALYZER
#define _ANALYZER

#include <iostream> 
#include <list>

#include "freeling.h"

namespace freeling {

// codes for input-output formats
typedef enum {TEXT,IDENT,TOKEN,SPLITTED,MORFO,TAGGED,SENSES,SHALLOW,PARSED,DEP,COREF,SEMGRAPH} AnalysisLevel;
// codes for tagging algorithms
typedef enum {NO_TAGGER,HMM,RELAX} TaggerAlgorithm;
// codes for dependency parsers
typedef enum {NO_DEP,TXALA,TREELER} DependencyParser;
// codes for sense annotation
typedef enum {NO_WSD,ALL,MFS,UKB} WSDAlgorithm;
// codes for ForceSelect
typedef enum {NO_FORCE,TAGGER,RETOK} ForceSelectStrategy;



class WINDLL analyzer {

 private:


   class analyzer_config_options {
     public:
       std::wstring Lang;
       std::wstring TOK_TokenizerFile;
       std::wstring SPLIT_SplitterFile;
       std::wstring MACO_Decimal, MACO_Thousand;
       std::wstring MACO_UserMapFile, MACO_LocutionsFile,   MACO_QuantitiesFile,
         MACO_AffixFile,   MACO_ProbabilityFile, MACO_DictionaryFile, 
         MACO_NPDataFile,  MACO_PunctuationFile, MACO_CompoundFile;      
       double MACO_ProbabilityThreshold;
       std::wstring PHON_PhoneticsFile;
       std::wstring NEC_NECFile;
       std::wstring SENSE_ConfigFile;
       std::wstring UKB_ConfigFile;
       std::wstring TAGGER_HMMFile;
       std::wstring TAGGER_RelaxFile;
       int TAGGER_RelaxMaxIter;
       double TAGGER_RelaxScaleFactor;
       double TAGGER_RelaxEpsilon;
       bool TAGGER_Retokenize;
       ForceSelectStrategy TAGGER_ForceSelect;
       std::wstring PARSER_GrammarFile;
       std::wstring DEP_TxalaFile;   
       std::wstring DEP_TreelerFile;   
       std::wstring COREF_CorefFile;
       std::wstring SEMGRAPH_SemGraphFile;
   };

   
   class analyzer_invoke_options {
     public:
       AnalysisLevel InputLevel, OutputLevel;

       bool MACO_UserMap, MACO_AffixAnalysis, MACO_MultiwordsDetection, 
         MACO_NumbersDetection, MACO_PunctuationDetection, 
         MACO_DatesDetection, MACO_QuantitiesDetection, 
         MACO_DictionarySearch, MACO_ProbabilityAssignment, MACO_CompoundAnalysis,
         MACO_NERecognition, MACO_RetokContractions;

       bool PHON_Phonetics;
       bool NEC_NEClassification;

       WSDAlgorithm SENSE_WSD_which;
       TaggerAlgorithm TAGGER_which;
       DependencyParser DEP_which;    
   };
   

   // we use pointers to the analyzers, so we
   // can create only those strictly necessary.
   tokenizer *tk;
   splitter *sp;
   maco *morfo;
   nec *neclass;
   senses *sens;
   ukb *dsb;
   POS_tagger *hmm;
   POS_tagger *relax;
   phonetics *phon;
   chart_parser *parser;
   dep_txala *deptxala;
   dep_treeler *deptreeler;
   relaxcor *corfc;
   semgraph_extract *sge;

   // store configuration options
   //   config *cfg;
   analyzer_invoke_options current_invoke_options;

   // remember splitter session
   splitter::session_id sp_id;

   // remember token offsets in plain text input
   unsigned long offs;
   // number of sentences processed (used to generate sentence id's)
   unsigned long nsentence;
   // words pending of being splitted in InputMode==CORPUS
   std::list<word> tokens; 
  
   template<class T> void do_analysis(T &doc) const;
   // tokenize and split text.
   void tokenize_split(const std::wstring &text, 
                       std::list<sentence> &ls, 
                       unsigned long &offs, 
                       std::list<word> &av, 
                       unsigned long &nsent, 
                       bool flush, 
                       splitter::session_id sp_ses) const;

 public:
   typedef analyzer_config_options config_options;
   typedef analyzer_invoke_options invoke_options;

   analyzer(const config_options &cfg);
   void set_current_invoke_options(const invoke_options &opt, bool check=true);
   const invoke_options& get_current_invoke_options() const;

   ~analyzer();
   void analyze(document &doc) const;
   void analyze(std::list<sentence> &ls) const;
   void analyze(const wstring &text, document &doc, bool parag=false) const;
   void analyze(const wstring &text, std::list<sentence> &ls, bool flush=false);
   // flush splitter buffer and analyze any pending text. 
   void flush_buffer(std::list<sentence> &ls);
 
   void reset_offset();
};



} // namespace

#endif