tokenizer.h

Go to the documentation of this file.

/******************************************************
 *  Presage, an extensible predictive text entry system
 *  ---------------------------------------------------
 *
 *  Copyright (C) 2008  Matteo Vescovi <matteo.vescovi@yahoo.co.uk>

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License along
    with this program; if not, write to the Free Software Foundation, Inc.,
    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
                                                                             *
                                                                **********(*)*/


#ifndef PRESAGE_TOKENIZER
#define PRESAGE_TOKENIZER

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <iostream>
#include <istream>
#include <string>
#include <assert.h>

class Tokenizer {
public:
    Tokenizer(std::istream& stream,
              const std::string   blankspaces,
              const std::string   separators  );
    virtual ~Tokenizer();

    virtual int countTokens() = 0;

    virtual bool hasMoreTokens() const = 0;
    
    virtual std::string nextToken() = 0;

    virtual double progress() const = 0;


    void blankspaceChars(const std::string);
    std::string blankspaceChars() const;

    void separatorChars(const std::string);
    std::string separatorChars() const;

    void lowercaseMode(const bool);
    bool lowercaseMode() const;

    std::string streamToString() const {
        std::streamoff offbackup = stream.tellg();
        std::string str;
        std::streamoff curroff = offbeg;
        stream.seekg(curroff);
        while (curroff < offend) {
            stream.clear();
            str.push_back(stream.peek());
            curroff++;
            stream.seekg(curroff);
        }
        stream.seekg(offbackup);
        return str;
    }
    
protected:
    class StreamGuard {
    public:
        StreamGuard(std::istream& so, std::streamoff& of) 
            : guardedStream(so) {
            currstate = guardedStream.rdstate();
            curroff   = guardedStream.tellg();
            guardedStream.seekg   (of       );
        }
        ~StreamGuard() {
            guardedStream.seekg   (curroff  );
            guardedStream.setstate(currstate);
        }

    private:
        std::istream&     guardedStream;
        std::ios::iostate currstate;
        std::streamoff    curroff;
    };

    std::istream&     stream;
    std::ios::iostate sstate;
    std::streamoff    offbeg;
    std::streamoff    offend;
    std::streamoff    offset;

    bool isBlankspace(const int character) const;
    bool isSeparator (const int character) const;

private:
    std::string blankspaces;
    std::string separators;

    bool lowercase;
};

#endif // PRESAGE_TOKENIZER