splitter.h

Go to the documentation of this file.

//
//    FreeLing - Open Source Language Analyzers
//
//    Copyright (C) 2014   TALP Research Center
//                         Universitat Politecnica de Catalunya
//
//    This library is free software; you can redistribute it and/or
//    modify it under the terms of the GNU Affero General Public
//    License as published by the Free Software Foundation; either
//    version 3 of the License, or (at your option) any later version.
//
//    This library is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//    Affero General Public License for more details.
//
//    You should have received a copy of the GNU Affero General Public
//    License along with this library; if not, write to the Free Software
//    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//    contact: Lluis Padro (padro@lsi.upc.es)
//             TALP Research Center
//             despatx C6.212 - Campus Nord UPC
//             08034 Barcelona.  SPAIN
//

#ifndef _SPLITTER
#define _SPLITTER

#include <map>
#include <set>

#include "freeling/windll.h"
#include "freeling/morfo/language.h"

namespace freeling {


  class WINDLL splitter {


  private:
    bool SPLIT_AllowBetweenMarkers;
    int SPLIT_MaxWords;
    std::set<std::wstring> starters;
    std::map<std::wstring,bool> enders;
    std::map<std::wstring,int> markers;

    bool end_of_sentence(std::list<word>::const_iterator, const std::list<word> &) const;

    // current state: Remember half-recognized sentences and open markers.
    class splitter_status {
      public:
        bool betweenMrk;
        int no_split_count;
        std::list<int> mark_type;
        std::list<std::wstring> mark_form;
        sentence buffer; 
        size_t nsentence;
    };

  public:
    splitter(const std::wstring &splitfile);
    ~splitter();

    typedef splitter_status* session_id;
    session_id open_session() const;
    void close_session(session_id ses) const;

    void split(session_id ses, const std::list<word> &lw, bool flush, std::list<sentence> &ls) const;

    std::list<sentence> split(session_id ses, const std::list<word> &ls, bool flush) const;

    // sessionless splitting.  
    // Equivalent to opening a session, split with flush=true, and close the session
    void split(const std::list<word> &lw, std::list<sentence> &ls) const;
    std::list<sentence> split(const std::list<word> &ls) const;
  };

} // namespace

#endif