idioma.h

Go to the documentation of this file.

//
//    Copyright (C) 2014   TALP Research Center
//                         Universitat Politecnica de Catalunya
//
//    This library is free software; you can redistribute it and/or
//    modify it under the terms of the GNU Affero General Public License
//    (GNU AGPL) as published by the Free Software Foundation; either
//    version 3 of the License, or (at your option) any later version.
//
//    This library is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
//    Affero General Public License for more details.
//
//    You should have received a copy of the GNU Affero General Public License 
//    along with this library; if not, write to the Free Software
//    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//    contact: Muntsa Padro (mpadro@lsi.upc.edu)
//             TALP Research Center
//             despatx Omega.S107 - Campus Nord UPC
//             08034 Barcelona.  SPAIN
//

//
//  idioma.h
//
//  Class that implements a Visible Markov Model 
//  to compute the probability that a text is 
//  written in a certain language.
// 

#ifndef _IDIOMA_H
#define _IDIOMA_H

#include <map>
#include <list>
#include <vector>
#include <string>

#include "freeling/morfo/smoothingLD.h"

namespace freeling {


  class idioma {  

  private:
    // language code
    std::wstring LangCode;

    std::map<std::wstring,double> count;

    wchar_t phantom;
    int order;
    double threshold;

    // smoother for ngram probabilities
    smoothingLD<std::wstring,wchar_t> *smooth;

    static std::wstring to_writable(wchar_t c);
    static std::wstring to_writable(const std::wstring &);

    static void initial_ngram(std::wistream &f, std::wstring &ngram, wchar_t &z, int ord, wchar_t ph);
    static void next_ngram(std::wistream &f, std::wstring &ngram, wchar_t &z);
 
  public:
    idioma(const std::wstring &);
    ~idioma();

    double sequence_probability(std::wistream &, size_t &) const;
    double compute_perplexity(const std::wstring &) const; 

    static void create_model(const std::wstring &modelFile,
                             std::wistream &f, 
                             const std::wstring &code, 
                             int order, 
                             wchar_t phantom);
      

    std::wstring get_language_code() const;
    double get_threshold() const;
  };

} // namespace

#endif