language.h

Go to the documentation of this file.

//
//    FreeLing - Open Source Language Analyzers
//
//    Copyright (C) 2014   TALP Research Center
//                         Universitat Politecnica de Catalunya
//
//    This library is free software; you can redistribute it and/or
//    modify it under the terms of the GNU Affero General Public
//    License as published by the Free Software Foundation; either
//    version 3 of the License, or (at your option) any later version.
//
//    This library is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//    Affero General Public License for more details.
//
//    You should have received a copy of the GNU Affero General Public
//    License along with this library; if not, write to the Free Software
//    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//    contact: Lluis Padro (padro@lsi.upc.es)
//             TALP Research Center
//             despatx C6.212 - Campus Nord UPC
//             08034 Barcelona.  SPAIN
//

#ifndef _LANGUAGE
#define _LANGUAGE

#include <string>
#include <list>
#include <vector>
#include <set>
#include <map>

#include "freeling/regexp.h"
#include "freeling/windll.h"
#include "freeling/tree.h"
#include "freeling/morfo/semgraph.h"

namespace freeling {

  class word; // predeclaration


  class WINDLL analysis {

  private:
    std::wstring lemma;
    std::wstring tag;
    double prob;
    double distance;
    std::list<std::pair<std::wstring,double> > senses;
    std::list<word> retok;

    // store which sequences --among the kbest proposed by 
    // the tagger-- contain this analysis
    std::set<int> selected_kbest;

  public:
    std::vector<std::wstring> user;

    analysis();
    analysis(const std::wstring &, const std::wstring &);
    analysis& operator=(const analysis&);

    void init(const std::wstring &l, const std::wstring &t);
    void set_lemma(const std::wstring &);
    void set_tag(const std::wstring &);
    void set_prob(double);
    void set_distance(double);
    void set_retokenizable(const std::list<word> &);

    bool has_prob() const;
    bool has_distance() const;
    const std::wstring& get_lemma() const;
    const std::wstring& get_tag() const;
    double get_prob() const;
    double get_distance() const;
    bool is_retokenizable() const;
    std::list<word>& get_retokenizable();
    const std::list<word>& get_retokenizable() const;

    const std::list<std::pair<std::wstring,double> > & get_senses() const;
    std::list<std::pair<std::wstring,double> > & get_senses();
    void set_senses(const std::list<std::pair<std::wstring,double> > &);
    // useful for java API
    std::wstring get_senses_string() const;

    // get the largest kbest sequence index the analysis is selected in.
    int max_kbest() const;
    // find out whether the analysis is selected in the tagger k-th best sequence
    bool is_selected(int k=0) const;
    // mark this analysis as selected in k-th best sequence
    void mark_selected(int k=0);
    // unmark this analysis as selected in k-th best sequence
    void unmark_selected(int k=0);

    bool operator>(const analysis &) const;
    bool operator<(const analysis &) const;
    bool operator==(const analysis &) const;
  };



  class WINDLL word : public std::list<analysis> {
  private:
    std::wstring form;
    std::wstring lc_form;
    std::wstring ph_form;
    std::list<word> multiword;
    bool ambiguous_mw;
    std::list<std::pair<std::wstring,int> > alternatives;
    unsigned long start, finish;
    bool in_dict;
    bool locked;
    void clone(const word &);
    size_t position;
 
    static const int SELECTED=0;
    static const int UNSELECTED=1;
    static const int ALL=2;
    static const std::wstring NOT_FOUND;

  public:
    // predeclarations
    class iterator; 
    class const_iterator; 

    std::vector<std::wstring> user;

    word();
    word(const std::wstring &);
    word(const std::wstring &, const std::list<word> &);
    word(const std::wstring &, const std::list<analysis> &, const std::list<word> &);
    word(const word &);
    word& operator=(const word&);

    void copy_analysis(const word &);
    int get_n_selected(int k=0) const;
    int get_n_unselected(int k=0) const;
    bool is_multiword() const;
    bool is_ambiguous_mw() const;
    void set_ambiguous_mw(bool);
    int get_n_words_mw() const;
    const std::list<word>& get_words_mw() const;
    const std::wstring& get_form() const;
    const std::wstring& get_lc_form() const;
    const std::wstring& get_ph_form() const;
    word::iterator selected_begin(int k=0);
    word::const_iterator selected_begin(int k=0) const;
    word::iterator selected_end(int k=0);
    word::const_iterator selected_end(int k=0) const;
    word::iterator unselected_begin(int k=0);
    word::const_iterator unselected_begin(int k=0) const;
    word::iterator unselected_end(int k=0);
    word::const_iterator unselected_end(int k=0) const;
    unsigned int num_kbest() const;
    const std::wstring& get_lemma(int k=0) const;
    const std::wstring& get_tag(int k=0) const;

    const std::list<std::pair<std::wstring,double> >& get_senses(int k=0) const;
    std::list<std::pair<std::wstring,double> >& get_senses(int k=0);
    // useful for java API
    std::wstring get_senses_string(int k=0) const;
    void set_senses(const std::list<std::pair<std::wstring,double> > &, int k=0);

    unsigned long get_span_start() const;
    unsigned long get_span_finish() const;

    bool found_in_dict() const;
    void set_found_in_dict(bool);
    bool has_retokenizable() const;
    void lock_analysis();
    bool is_locked() const;

    void add_alternative(const std::wstring &, int);
    void set_alternatives(const std::list<std::pair<std::wstring,int> > &);
    void clear_alternatives();
    bool has_alternatives() const;
    const std::list<std::pair<std::wstring,int> >& get_alternatives() const;
    std::list<std::pair<std::wstring,int> >& get_alternatives();
    std::list<std::pair<std::wstring,int> >::iterator alternatives_begin();
    std::list<std::pair<std::wstring,int> >::iterator alternatives_end();
    std::list<std::pair<std::wstring,int> >::const_iterator alternatives_begin() const;
    std::list<std::pair<std::wstring,int> >::const_iterator alternatives_end() const;

    void add_analysis(const analysis &);
    void set_analysis(const analysis &);
    void set_analysis(const std::list<analysis> &);
    void set_form(const std::wstring &);
    void set_ph_form(const std::wstring &);
    void set_span(unsigned long, unsigned long);

    // get/set word position
    void set_position(size_t);
    size_t get_position() const;

    bool find_tag_match(const freeling::regexp &) const;

    int get_n_analysis() const;
    void unselect_all_analysis(int k=0);
    void select_all_analysis(int k=0);
    void select_analysis(word::iterator, int k=0);
    void unselect_analysis(word::iterator, int k=0);
    std::list<analysis> get_analysis() const;
    word::iterator analysis_begin();
    word::const_iterator analysis_begin() const;
    word::iterator analysis_end();
    word::const_iterator analysis_end() const;

    class WINDLL iterator : public std::list<analysis>::iterator {
      friend class word::const_iterator;
    private:
      std::list<analysis>::iterator ibeg;
      std::list<analysis>::iterator iend;
      int type;
      int kbest;

    public:
      iterator();
      iterator(const word::iterator &);
      iterator(const std::list<analysis>::iterator &);
      iterator(const std::list<analysis>::iterator &, 
               const std::list<analysis>::iterator &, 
               const std::list<analysis>::iterator &, int,int k=0);  
      iterator& operator++();
      iterator operator++(int);
    };
  
    class WINDLL const_iterator : public std::list<analysis>::const_iterator {
    private:
      std::list<analysis>::const_iterator ibeg;
      std::list<analysis>::const_iterator iend;
      int type;
      int kbest;

    public:
      const_iterator();
      const_iterator(const word::const_iterator &);
      const_iterator(const word::iterator &);
      const_iterator(const std::list<analysis>::const_iterator &);
      const_iterator(const std::list<analysis>::iterator &);
      const_iterator(const std::list<analysis>::const_iterator &,
                     const std::list<analysis>::const_iterator &, 
                     const std::list<analysis>::const_iterator &, int, int k=0);
      const_iterator& operator++();  
      const_iterator operator++(int);  
    };

  };




  class WINDLL node {
  protected:
    std::wstring nodeid;
    bool head;
    int chunk;
    std::wstring label;
    word * w;

  public:
    std::vector<std::wstring> user;
    node();
    node(const std::wstring &);
    const std::wstring& get_node_id() const;
    void set_node_id(const std::wstring &);
    const std::wstring& get_label() const;
    bool has_word() const;
    const word & get_word() const;
    word & get_word();
    void set_label(const std::wstring &);
    void set_word(word &);
    bool is_head() const;
    void set_head(const bool);
    bool is_chunk() const;
    void set_chunk(const int);
    int get_chunk_ord() const;

  };


  class WINDLL parse_tree : public tree<node> {
  private:
    // access nodes by id
    std::map<std::wstring,parse_tree::iterator> node_index;
    // acces leaf nodes by word position
    std::vector<parse_tree::iterator> word_index;

  public:
    parse_tree();
    parse_tree(parse_tree::const_iterator p);
    parse_tree(const node &);

    void build_node_index(const std::wstring &);
    void rebuild_node_index();
    parse_tree::const_iterator get_node_by_id(const std::wstring &) const;
    parse_tree::const_iterator get_node_by_pos(size_t) const;
    parse_tree::iterator get_node_by_id(const std::wstring &);
    parse_tree::iterator get_node_by_pos(size_t);

    static const word& get_head_word(parse_tree::const_iterator);
    static const std::wstring& get_head_label(parse_tree::const_iterator);
    static int get_head_position(parse_tree::const_iterator);

    static bool C_commands(parse_tree::const_iterator, parse_tree::const_iterator);
    
    static void PrintTree(parse_tree::const_iterator n, int k, int depth);

    parse_tree::const_iterator get_subsuming_node(size_t i, size_t j) const;
    parse_tree::iterator get_subsuming_node(size_t i, size_t j);
    parse_tree::const_iterator get_left_subsuming_node(size_t i, size_t j) const;
    parse_tree::iterator get_left_subsuming_node(size_t i, size_t j);
    parse_tree::const_iterator get_right_subsuming_node(size_t i, size_t j) const;
    parse_tree::iterator get_right_subsuming_node(size_t i, size_t j);
  };



  class WINDLL depnode : public node {

  private:
    parse_tree::iterator link;

  public:
    depnode();
    depnode(const std::wstring &);
    depnode(const node &);

    void set_link(const parse_tree::iterator);
    parse_tree::iterator get_link();
    parse_tree::const_iterator get_link() const;
  };


  class WINDLL dep_tree :  public tree<depnode> {

  private:
    // acces nodes by word position
    std::vector<dep_tree::iterator> word_index;

  public:
    dep_tree();
    dep_tree(const depnode &);

    dep_tree::const_iterator get_node_by_pos(size_t) const;
    dep_tree::iterator get_node_by_pos(size_t);
    void rebuild_node_index();

    static size_t get_first_word(dep_tree::const_iterator);
    static size_t get_last_word(dep_tree::const_iterator);

    static void PrintDepTree(dep_tree::const_iterator n, int depth);
  };




  class processor_status {
  public:
    processor_status();
    virtual ~processor_status() {};
  };




  class WINDLL argument  {
    private:
      int position;
      std::wstring role;

    public:
      static const std::wstring EMPTY_ROLE;

      argument();
      ~argument();
      argument(int p, const std::wstring &r);
      
      int get_position() const;
      const std::wstring& get_role() const;
  };


  class WINDLL predicate : public std::vector<argument> {

    private:
      // index to find arguments by word position
      std::map<int,int> arg_index;
      // position of the predicate head in the sentence
      int position;
      // propbank sense of the predicate
      std::wstring sense;

    public:

      predicate();
      ~predicate();
      predicate(int p, const std::wstring &s);
      predicate(const predicate &);
      predicate& operator=(const predicate&);

      const std::wstring& get_sense() const;
      int get_position() const;
      bool has_argument(int p) const;
      void add_argument(int p, const std::wstring &r);
      const argument & get_argument_by_pos(int p) const;

  };



  class WINDLL sentence : public std::list<word> {

  public:
    typedef std::vector<predicate> predicates;

  private:
    // sentence identifier, in case user application wants to set it.
    std::wstring sent_id;
    // vector with pointers to sentence words, for fast access by position
    std::vector<word*> wpos; 
    // remember if it is PoS tagged
    bool tagged;
    // parse tree (if sentence parsed)
    std::map<int,parse_tree> pts;
    // dependencey tree (if sentence dep. parsed)
    std::map<int,dep_tree> dts;
    // clone sentence (used by assignment/copy constructors)
    void clone(const sentence &);
    // stack processing status for processor currently analyzing the sentence
    // (there might be a hierarchy of embeeded processors, thus the stack)
    std::list<processor_status*> status;
    // store map of predicates by position
    predicates preds;
    // index to access predicates by word position
    std::map<int,int> pred_index;

  public:

    sentence();
    ~sentence();
    sentence(const std::list<word>&);
    sentence(const sentence &);
    sentence& operator=(const sentence&);
    const word& operator[](size_t) const;
    word& operator[](size_t);
    unsigned int num_kbest() const;
    void push_back(const word &);
    void rebuild_word_index();
     
    void clear();

    void set_sentence_id(const std::wstring &);
    const std::wstring& get_sentence_id() const;

    void set_is_tagged(bool);
    bool is_tagged() const;

    void set_parse_tree(const parse_tree &, int k=0);
    parse_tree & get_parse_tree(int k=0);
    const parse_tree & get_parse_tree(int k=0) const;
    bool is_parsed() const;

    void set_dep_tree(const dep_tree &, int k=0);
    dep_tree & get_dep_tree(int k=0);
    const dep_tree & get_dep_tree(int k=0) const;
    bool is_dep_parsed() const;

    processor_status* get_processing_status();
    const processor_status* get_processing_status() const;
    void set_processing_status(processor_status *);
    void clear_processing_status();

    std::vector<word> get_words() const;
    sentence::iterator words_begin();
    sentence::const_iterator words_begin() const;
    sentence::iterator words_end();
    sentence::const_iterator words_end() const;

    // obtain iterator to a word given its position
    sentence::iterator get_word_iterator(const word &w);
    sentence::const_iterator get_word_iterator(const word &w) const;

    void add_predicate(const predicate &pr);
    bool is_predicate(int p) const;
    int get_predicate_number(int p) const;
    int get_predicate_position(int n) const;
    const predicate& get_predicate_by_pos(int n) const;
    const predicate& get_predicate_by_number(int n) const;
    const predicates & get_predicates() const;
  };


  class WINDLL paragraph : public std::list<sentence> {
  private:
    std::wstring par_id;
  public:
    paragraph();
    paragraph(const std::list<sentence> &x);
    void set_paragraph_id(const std::wstring &);
    const std::wstring & get_paragraph_id() const;
  };

  
  class WINDLL mention {

  public:
    typedef enum {PROPER_NOUN, PRONOUN, NOUN_PHRASE, COMPOSITE, VERB_PHRASE} mentionType;
    // NE type NOTPER is the supertype of ORG, GEO and OTHER
    typedef enum {PER, MALE, FEMALE, NOTPER, ORG, GEO, TIME, DATE, MONEY, OTHER} SEMmentionType;

  private:

    int id;
    mentionType mType;
    int sent;
    paragraph::const_iterator s;
    bool initial;
    bool single_subsumtion;
    parse_tree::const_iterator ptree;
    int posBegin;
    int posEnd;
    sentence::const_iterator itBegin;
    sentence::const_iterator itEnd;
    sentence::const_iterator itHead;
    int chain;


    void clone(const mention &);
    static void set_tokens(parse_tree::const_iterator, int&, sentence::const_iterator&);
    static void set_iterators(sentence::const_iterator, sentence::const_iterator, const parse_tree&, sentence::const_iterator&, parse_tree::const_iterator&);

  public:

    mention(int, int, paragraph::const_iterator, parse_tree::const_iterator, int, sentence::const_iterator);
    mention(int, int, paragraph::const_iterator, sentence::const_iterator, sentence::const_iterator);
    mention(const mention &);
    mention& operator=(const mention&);

    bool operator<(const mention &m) const;

    void set_id(int);
    void set_type(mentionType);
    void set_initial(bool);
    void set_group(int);
    void subsumed_with_no_verb(bool b=false);

    int get_id() const;
    int get_n_sentence() const;
    paragraph::const_iterator get_sentence() const;
    int get_pos_begin() const;
    int get_pos_end() const;
    sentence::const_iterator get_it_begin() const;
    sentence::const_iterator get_it_end() const;
    sentence::const_iterator get_it_head() const;
    mentionType get_type() const;
    int get_group() const;
    bool is_type(mentionType) const;
    bool is_initial() const;
    bool is_subsumed_with_no_verb() const;
    parse_tree::const_iterator get_ptree() const;
    const word& get_head() const;
    std::wstring value(int lc=0) const;    
  };




  class WINDLL document : public std::list<paragraph> {

  private:
    static const size_t DIM = 500;

    paragraph title;

    // potentially coreferring mentions
    std::vector<mention> mentions;
    // set of mention_ids belonging to the same group
    std::multimap<int,int> group2mentions;
    // ids  (non correlative) for existing groups
    std::list<int> groups;

    // semantic graph
    semgraph::semantic_graph sem_graph;

  public:
    document();
    ~document();

    bool is_parsed() const;
    bool is_dep_parsed() const;

    void add_mention(const mention &m);


    // count number of words in the document
    int get_num_words() const;

    int get_num_groups() const;
    const std::list<int> & get_groups() const;

    std::vector<mention>::const_iterator begin_mentions() const;
    std::vector<mention>::iterator begin_mentions();
    std::vector<mention>::const_iterator end_mentions() const;
    std::vector<mention>::iterator end_mentions();

    const mention& get_mention(int) const;


    std::list<int> get_coref_id_mentions(int) const;


    const semgraph::semantic_graph & get_semantic_graph() const;
    semgraph::semantic_graph & get_semantic_graph();
  };

} // namespace

#endif