Create a C++ Tokenizer class. This Tokenizer class will consist of both a header
ID: 3902556 • Letter: C
Question
Create a C++ Tokenizer class. This Tokenizer class will consist of both a header and a class file. You will create the prog2_1.hpp and prog2_1.cpp This Tokenizer class will have four public methods. First, the constructor, which will setup/create any internal data structures you need. Second, the destructor, which will delete any internal data structures you created during the lifetime of the object. Third will be a void function that takes a single string argument name Tokeni This Tokenize function will tokenize the input string on the space character. Only the following tokens are valid: push, pop, add, sub, mul, div, mod, skip, save, get and any valid integer If an input string contains a potential token that is not one of those values the function should throw a std::exception with the message "Unexpected token:Explanation / Answer
Given below is the code for the question.
Please do rate the answer if it was helpful. Thank you
Tokenizer.hpp
=============
#ifndef Tokenizer_hpp
#define Tokenizer_hpp
#include <vector>
#include <string>
using namespace std;
class Tokenizer
{
private:
vector< vector<string> > tokensQueue;
bool isValidToken(string s);
public:
Tokenizer(){}
~Tokenizer(){}
void Tokenize(string s);
vector<string> GetTokens();
};
#endif /* Tokenizer_hpp */
Tokenizer.cpp
=============
#include "Tokenizer.hpp"
#include <exception>
using namespace std;
void Tokenizer::Tokenize(string s)
{
size_t start = 0;
size_t end;
vector<string> tokens;
while(true)
{
end = s.find(" ", start);
string t;
if(end != string::npos)
{
t = s.substr(start, end - start);
start = end + 1;
}
else
t = s.substr(start);
if(isValidToken(t))
tokens.push_back(t);
else
throw runtime_error("Unexpected token: " + t);
if(end == string::npos)
break;
}
tokensQueue.push_back(tokens);
}
vector<string> Tokenizer::GetTokens()
{
if(tokensQueue.empty())
throw runtime_error("No tokens");
vector<string> tokens = tokensQueue[0];
tokensQueue.erase(tokensQueue.begin()); //delete the first item
return tokens;
}
bool Tokenizer::isValidToken(std::string s)
{
string valid_tokens[] = {"push", "pop", "add", "sub", "mul", "div", "mod", "skip","save", "get"};
for(int i = 0; i < 10; i++)
{
if(s == valid_tokens[i])
return true;
}
//not one of those tokens, check if it is integer
for(int i = 0; i < s.length(); i++)
{
//check if its not a digit
if(s[i] < '0' || s[i] >'9')
return false;
}
return true;
}
Test.cpp
=======
#include <iostream>
#include "Tokenizer.hpp"
void print(const vector<string> &t)
{
for(int i = 0; i < t.size(); i++)
cout << t[i] << " ";
cout << endl;
}
int main()
{
Tokenizer t;
t.Tokenize("push 10");
t.Tokenize("push 20");
t.Tokenize("pop");
print(t.GetTokens());
print(t.GetTokens());
print(t.GetTokens());
}
output
=====
push 10
push 20
pop