OpenASIP 2.2
Loading...
Searching...
No Matches
Classes | Public Types | Public Member Functions | Static Public Member Functions | Private Types | Private Attributes | List of all members
TokenizerData Class Reference

#include <OperationDAGLanguageParser.hh>

Collaboration diagram for TokenizerData:
Collaboration graph

Classes

class  Token
 
class  TokenTreeNode
 

Public Types

enum  OperationID {
  IDENTIFIER , STRING_LITERAL , INTEGER_LITERAL , FLOAT_LITERAL ,
  PRIMARY_EXPRESSION , UNARY_EXPRESSION , UNARY_ADDROF , UNARY_STAR ,
  UNARY_PLUS , UNARY_MINUS , UNARY_TILDE , UNARY_BANG ,
  POSTFIX_EXPRESSION , CAST_EXPRESSION , MULTIPLICATIVE_EXPRESSION , ADDITIVE_EXPRESSION ,
  SHIFT_EXPRESSION , RELATIONAL_EXPRESSION , EQUALITY_EXPRESSION , AND_EXPRESSION ,
  EXCLUSIVE_EXPRESSION , INCLUSIVE_EXPRESSION , LOGICAL_AND_EXPRESSION , LOGICAL_OR_EXPRESSION ,
  CONDITIONAL_EXPRESSION , ASSIGNMENT_EXPRESSION , ASSIGNMENT_OPERATOR , EXPRESSION ,
  CONSTANT_EXPRESSION , DECLARATION , DECLARATION_SPECIFIERS , INIT_DECLARATOR ,
  STORAGE_CLASS_SPECIFIER , TYPE_SPECIFIER , STRUCT_OR_UNION_SPECIFIER , STRUCT_OR_UNION ,
  STRUCT_DECLARATION , STRUCT_DECLARATOR , ENUM_SPECIFIER , ENUMERATOR ,
  CONST_QUALIFER , VOLATILE_QUALIFER , TYPE_QUALIFER , DECLARATOR ,
  INITIALIZER , DIRECT_DECLARATOR , POINTER , PARAMETER_TYPE_LIST ,
  PARAMETER_LIST , PARAMETER_DECLARATION , IDENTIFIER_LIST , TYPE_NAME ,
  ABSTRACT_DECLARATOR , DIRECT_ABSTRACT_DECLARATOR , INITIALIZER_LIST , STATEMENT ,
  LABELED_STATEMENT , COMPOUND_STATEMENT , STATEMENT_LIST , EXPRESSION_STATEMENT ,
  SELECTION_STATEMENT , ITERATION_STATEMENT , JUMP_STATEMENT , FUNCTION_DEFINITION ,
  EXTERNAL_DECLARATION , INVALID_TOKEN
}
 

Public Member Functions

 TokenizerData ()
 
 ~TokenizerData ()
 
void addToken (const char *start, const char *end, OperationID id, std::string strVal)
 
void showTokenizedSource ()
 
const TokenTreeNode * tokenTree () const
 

Static Public Member Functions

static std::string idString (OperationID id)
 

Private Types

typedef std::map< std::pair< const char *, const char * >, Token * > TokenContainerType
 

Private Attributes

TokenContainerType parsedTokens_
 
TokenTreeNode root
 

Detailed Description

Class which is used by Spirit parser to create tokenized tree form of the source code.

Definition at line 148 of file OperationDAGLanguageParser.hh.

Member Typedef Documentation

◆ TokenContainerType

typedef std::map< std::pair<const char*, const char*>, Token*> TokenizerData::TokenContainerType
private

Definition at line 705 of file OperationDAGLanguageParser.hh.

Member Enumeration Documentation

◆ OperationID

enum TokenizerData::OperationID

List of different type tokens.

Enumerator
IDENTIFIER 
STRING_LITERAL 
INTEGER_LITERAL 
FLOAT_LITERAL 
PRIMARY_EXPRESSION 
UNARY_EXPRESSION 
UNARY_ADDROF 
UNARY_STAR 
UNARY_PLUS 
UNARY_MINUS 
UNARY_TILDE 
UNARY_BANG 
POSTFIX_EXPRESSION 
CAST_EXPRESSION 
MULTIPLICATIVE_EXPRESSION 
ADDITIVE_EXPRESSION 
SHIFT_EXPRESSION 
RELATIONAL_EXPRESSION 
EQUALITY_EXPRESSION 
AND_EXPRESSION 
EXCLUSIVE_EXPRESSION 
INCLUSIVE_EXPRESSION 
LOGICAL_AND_EXPRESSION 
LOGICAL_OR_EXPRESSION 
CONDITIONAL_EXPRESSION 
ASSIGNMENT_EXPRESSION 
ASSIGNMENT_OPERATOR 
EXPRESSION 
CONSTANT_EXPRESSION 
DECLARATION 
DECLARATION_SPECIFIERS 
INIT_DECLARATOR 
STORAGE_CLASS_SPECIFIER 
TYPE_SPECIFIER 
STRUCT_OR_UNION_SPECIFIER 
STRUCT_OR_UNION 
STRUCT_DECLARATION 
STRUCT_DECLARATOR 
ENUM_SPECIFIER 
ENUMERATOR 
CONST_QUALIFER 
VOLATILE_QUALIFER 
TYPE_QUALIFER 
DECLARATOR 
INITIALIZER 
DIRECT_DECLARATOR 
POINTER 
PARAMETER_TYPE_LIST 
PARAMETER_LIST 
PARAMETER_DECLARATION 
IDENTIFIER_LIST 
TYPE_NAME 
ABSTRACT_DECLARATOR 
DIRECT_ABSTRACT_DECLARATOR 
INITIALIZER_LIST 
STATEMENT 
LABELED_STATEMENT 
COMPOUND_STATEMENT 
STATEMENT_LIST 
EXPRESSION_STATEMENT 
SELECTION_STATEMENT 
ITERATION_STATEMENT 
JUMP_STATEMENT 
FUNCTION_DEFINITION 
EXTERNAL_DECLARATION 
INVALID_TOKEN 

Definition at line 154 of file OperationDAGLanguageParser.hh.

154 {
155 IDENTIFIER,
156 STRING_LITERAL,
157 INTEGER_LITERAL,
158 FLOAT_LITERAL,
159 PRIMARY_EXPRESSION,
160 UNARY_EXPRESSION,
161 UNARY_ADDROF,
162 UNARY_STAR,
163 UNARY_PLUS,
164 UNARY_MINUS,
165 UNARY_TILDE,
166 UNARY_BANG,
167 POSTFIX_EXPRESSION,
168 CAST_EXPRESSION,
169 MULTIPLICATIVE_EXPRESSION,
170 ADDITIVE_EXPRESSION,
171 SHIFT_EXPRESSION,
172 RELATIONAL_EXPRESSION,
173 EQUALITY_EXPRESSION,
174 AND_EXPRESSION,
175 EXCLUSIVE_EXPRESSION,
176 INCLUSIVE_EXPRESSION,
177 LOGICAL_AND_EXPRESSION,
178 LOGICAL_OR_EXPRESSION,
179 CONDITIONAL_EXPRESSION,
180 ASSIGNMENT_EXPRESSION,
181 ASSIGNMENT_OPERATOR,
182 EXPRESSION,
183 CONSTANT_EXPRESSION,
184 DECLARATION,
185 DECLARATION_SPECIFIERS,
186 INIT_DECLARATOR,
187 STORAGE_CLASS_SPECIFIER,
188 TYPE_SPECIFIER,
189 STRUCT_OR_UNION_SPECIFIER,
190 STRUCT_OR_UNION,
191 STRUCT_DECLARATION,
192 STRUCT_DECLARATOR,
193 ENUM_SPECIFIER,
194 ENUMERATOR,
195 CONST_QUALIFER,
196 VOLATILE_QUALIFER,
197 TYPE_QUALIFER,
198 DECLARATOR,
199 INITIALIZER,
200 DIRECT_DECLARATOR,
201 POINTER,
202 PARAMETER_TYPE_LIST,
203 PARAMETER_LIST,
204 PARAMETER_DECLARATION,
205 IDENTIFIER_LIST,
206 TYPE_NAME,
207 ABSTRACT_DECLARATOR,
208 DIRECT_ABSTRACT_DECLARATOR,
209 INITIALIZER_LIST,
210 STATEMENT,
211 LABELED_STATEMENT,
212 COMPOUND_STATEMENT,
213 STATEMENT_LIST,
214 EXPRESSION_STATEMENT,
215 SELECTION_STATEMENT,
216 ITERATION_STATEMENT,
217 JUMP_STATEMENT,
218 FUNCTION_DEFINITION,
219 EXTERNAL_DECLARATION,
220 INVALID_TOKEN
221 };

Constructor & Destructor Documentation

◆ TokenizerData()

TokenizerData::TokenizerData ( )
inline

Definition at line 613 of file OperationDAGLanguageParser.hh.

613{}

◆ ~TokenizerData()

TokenizerData::~TokenizerData ( )
inline

Deletes all the parsed data.

Definition at line 618 of file OperationDAGLanguageParser.hh.

618 {
619 while (!parsedTokens_.empty()) {
620 delete (parsedTokens_.begin()->second);
621 parsedTokens_.erase(parsedTokens_.begin());
622 }
623 // TODO: delete TokenTreeNodes
624 }
TokenizerData::parsedTokens_
TokenContainerType parsedTokens_
Definition OperationDAGLanguageParser.hh:707

References parsedTokens_.

Member Function Documentation

◆ addToken()

void TokenizerData::addToken ( const char *  start,
const char *  end,
OperationID  id,
std::string  strVal 
)
inline

Creates new token based on parsed data or updates new type info for old token.

Parameters
startStart position of token.
endEnd position of token.
idType of token.
strValString of token.

Definition at line 636 of file OperationDAGLanguageParser.hh.

637 {
638
639 unsigned long startPos, endPos;
640
641 if (parsedTokens_.empty()) {
642 startPos = 0;
643 endPos = end - start;
644 } else {
645 // in parsedTokens_.begin()->first.second is actually stored value
646 // (-start) of first parsed token. So we can add it to current
647 // start address for getting relative end and start positions.
648 startPos =
649 reinterpret_cast<unsigned long>(start) +
650 reinterpret_cast<unsigned long>(
651 parsedTokens_.begin()->first.second);
652
653 endPos =
654 reinterpret_cast<unsigned long>(end) +
655 reinterpret_cast<unsigned long>(
656 parsedTokens_.begin()->first.second);
657 }
658
659 // key for map is selected (end, -start) for automatically organizase
660 // tokens
661 std::pair<const char*, const char*>
662 tokenKey(end, (const char*)(-reinterpret_cast<unsigned long>(start)));
663
664 if (parsedTokens_.find(tokenKey) == parsedTokens_.end()) {
665 parsedTokens_[tokenKey] = new Token(startPos, endPos, strVal);
666 root.addToTokenTree(*parsedTokens_[tokenKey]);
667 }
668
669 parsedTokens_[tokenKey]->setProperty(id);
670 }

References TokenizerData::TokenTreeNode::addToTokenTree(), parsedTokens_, and root.

Referenced by TokenizerActor::operator()().

Here is the call graph for this function:

◆ idString()

static std::string TokenizerData::idString ( OperationID  id)
inlinestatic

Prints string value for each token.

Parameters
idToken type.
Returns
String for token type.

Definition at line 229 of file OperationDAGLanguageParser.hh.

229 {
230 switch (id) {
231 case IDENTIFIER: return "IDENTIFIER:";
232 case STRING_LITERAL: return "STRING_LITERAL:";
233 case INTEGER_LITERAL: return "INTEGER_LITERAL:";
234 case FLOAT_LITERAL: return "FLOAT_LITERAL:";
235 case PRIMARY_EXPRESSION: return "PRIMARY_EXPRESSION:";
236 case UNARY_EXPRESSION: return "UNARY_EXPRESSION:";
237 case UNARY_ADDROF: return "UNARY_ADDROF:";
238 case UNARY_STAR: return "UNARY_STAR:";
239 case UNARY_PLUS: return "UNARY_PLUS:";
240 case UNARY_MINUS: return "UNARY_MINUS:";
241 case UNARY_TILDE: return "UNARY_TILDE:";
242 case UNARY_BANG: return "UNARY_BANG:";
243 case POSTFIX_EXPRESSION: return "POSTFIX_EXPRESSION:";
244 case CAST_EXPRESSION: return "CAST_EXPRESSION:";
245 case MULTIPLICATIVE_EXPRESSION: return "MULTIPLICATIVE_EXPRESSION:";
246 case ADDITIVE_EXPRESSION: return "ADDITIVE_EXPRESSION:";
247 case SHIFT_EXPRESSION: return "SHIFT_EXPRESSION:";
248 case RELATIONAL_EXPRESSION: return "RELATIONAL_EXPRESSION:";
249 case EQUALITY_EXPRESSION: return "EQUALITY_EXPRESSION:";
250 case AND_EXPRESSION: return "AND_EXPRESSION:";
251 case EXCLUSIVE_EXPRESSION: return "EXCLUSIVE_EXPRESSION:";
252 case INCLUSIVE_EXPRESSION: return "INCLUSIVE_EXPRESSION:";
253 case LOGICAL_AND_EXPRESSION: return "LOGICAL_AND_EXPRESSION:";
254 case LOGICAL_OR_EXPRESSION: return "LOGICAL_OR_EXPRESSION:";
255 case CONDITIONAL_EXPRESSION: return "CONDITIONAL_EXPRESSION:";
256 case ASSIGNMENT_EXPRESSION: return "ASSIGNMENT_EXPRESSION:";
257 case ASSIGNMENT_OPERATOR: return "ASSIGNMENT_OPERATOR:";
258 case EXPRESSION: return "EXPRESSION:";
259 case CONSTANT_EXPRESSION: return "CONSTANT_EXPRESSION:";
260 case DECLARATION: return "DECLARATION:";
261 case DECLARATION_SPECIFIERS: return "DECLARATION_SPECIFIERS:";
262 case INIT_DECLARATOR: return "INIT_DECLARATOR:";
263 case STORAGE_CLASS_SPECIFIER: return "STORAGE_CLASS_SPECIFIER:";
264 case TYPE_SPECIFIER: return "TYPE_SPECIFIER:";
265 case STRUCT_OR_UNION_SPECIFIER: return "STRUCT_OR_UNION_SPECIFIER:";
266 case STRUCT_OR_UNION: return "STRUCT_OR_UNION:";
267 case STRUCT_DECLARATION: return "STRUCT_DECLARATION:";
268 case STRUCT_DECLARATOR: return "STRUCT_DECLARATOR:";
269 case ENUM_SPECIFIER: return "ENUM_SPECIFIER:";
270 case ENUMERATOR: return "ENUMERATOR:";
271 case CONST_QUALIFER: return "CONST_QUALIFER:";
272 case VOLATILE_QUALIFER: return "VOLATILE_QUALIFER:";
273 case TYPE_QUALIFER: return "TYPE_QUALIFER:";
274 case DECLARATOR: return "DECLARATOR:";
275 case INITIALIZER: return "INITIALIZER:";
276 case DIRECT_DECLARATOR: return "DIRECT_DECLARATOR:";
277 case POINTER: return "POINTER:";
278 case PARAMETER_TYPE_LIST: return "PARAMETER_TYPE_LIST:";
279 case PARAMETER_LIST: return "PARAMETER_LIST:";
280 case PARAMETER_DECLARATION: return "PARAMETER_DECLARATION:";
281 case IDENTIFIER_LIST: return "IDENTIFIER_LIST:";
282 case TYPE_NAME: return "TYPE_NAME:";
283 case ABSTRACT_DECLARATOR: return "ABSTRACT_DECLARATOR:";
284 case DIRECT_ABSTRACT_DECLARATOR: return "DIRECT_ABSTRACT_DECLARATOR:";
285 case INITIALIZER_LIST: return "INITIALIZER_LIST:";
286 case STATEMENT: return "STATEMENT:";
287 case LABELED_STATEMENT: return "LABELED_STATEMENT:";
288 case COMPOUND_STATEMENT: return "COMPOUND_STATEMENT:";
289 case STATEMENT_LIST: return "STATEMENT_LIST:";
290 case EXPRESSION_STATEMENT: return "EXPRESSION_STATEMENT:";
291 case SELECTION_STATEMENT: return "SELECTION_STATEMENT:";
292 case ITERATION_STATEMENT: return "ITERATION_STATEMENT:";
293 case JUMP_STATEMENT: return "JUMP_STATEMENT:";
294 case FUNCTION_DEFINITION: return "FUNCTION_DEFINITION:";
295 case EXTERNAL_DECLARATION: return "EXTERNAL_DECLARATION:";
296 case INVALID_TOKEN: return "INVALID_TOKEN:";
297 default: return "UNKNOWN_ID:";
298 }
299 }

References ABSTRACT_DECLARATOR, ADDITIVE_EXPRESSION, AND_EXPRESSION, ASSIGNMENT_EXPRESSION, ASSIGNMENT_OPERATOR, CAST_EXPRESSION, COMPOUND_STATEMENT, CONDITIONAL_EXPRESSION, CONST_QUALIFER, CONSTANT_EXPRESSION, DECLARATION, DECLARATION_SPECIFIERS, DECLARATOR, DIRECT_ABSTRACT_DECLARATOR, DIRECT_DECLARATOR, ENUM_SPECIFIER, ENUMERATOR, EQUALITY_EXPRESSION, EXCLUSIVE_EXPRESSION, EXPRESSION, EXPRESSION_STATEMENT, EXTERNAL_DECLARATION, FLOAT_LITERAL, FUNCTION_DEFINITION, IDENTIFIER, IDENTIFIER_LIST, INCLUSIVE_EXPRESSION, INIT_DECLARATOR, INITIALIZER, INITIALIZER_LIST, INTEGER_LITERAL, INVALID_TOKEN, ITERATION_STATEMENT, JUMP_STATEMENT, LABELED_STATEMENT, LOGICAL_AND_EXPRESSION, LOGICAL_OR_EXPRESSION, MULTIPLICATIVE_EXPRESSION, PARAMETER_DECLARATION, PARAMETER_LIST, PARAMETER_TYPE_LIST, POINTER, POSTFIX_EXPRESSION, PRIMARY_EXPRESSION, RELATIONAL_EXPRESSION, SELECTION_STATEMENT, SHIFT_EXPRESSION, STATEMENT, STATEMENT_LIST, STORAGE_CLASS_SPECIFIER, STRING_LITERAL, STRUCT_DECLARATION, STRUCT_DECLARATOR, STRUCT_OR_UNION, STRUCT_OR_UNION_SPECIFIER, TYPE_NAME, TYPE_QUALIFER, TYPE_SPECIFIER, UNARY_ADDROF, UNARY_BANG, UNARY_EXPRESSION, UNARY_MINUS, UNARY_PLUS, UNARY_STAR, UNARY_TILDE, and VOLATILE_QUALIFER.

Referenced by showTokenizedSource(), and TokenizerData::TokenTreeNode::toStr().

◆ showTokenizedSource()

void TokenizerData::showTokenizedSource ( )
inline

Shows source code parsed to tokens.

Definition at line 675 of file OperationDAGLanguageParser.hh.

675 {
676
677 for (TokenContainerType::iterator i = parsedTokens_.begin();
678 i != parsedTokens_.end(); i++) {
679
680 std::cerr << std::hex
681 << std::setw(4) << i->second->start_
682 << " - "
683 << std::setw(4) << i->second->end_
684
685 << " : "
686 << std::setw(30) << idString(i->second->type_)
687
688 << " : "
689 << i->second->strValue_
690 << std::endl;
691 }
692 }
TokenizerData::idString
static std::string idString(OperationID id)
Definition OperationDAGLanguageParser.hh:229

References idString(), and parsedTokens_.

Here is the call graph for this function:

◆ tokenTree()

const TokenTreeNode * TokenizerData::tokenTree ( ) const
inline

Returns root node of token tree.

Returns
Root node of token tree.

Definition at line 699 of file OperationDAGLanguageParser.hh.

699 {
700 return &root;
701 }

References root.

Referenced by OperationDAGConverter::createDAG().

Member Data Documentation

◆ parsedTokens_

TokenContainerType TokenizerData::parsedTokens_
private

Definition at line 707 of file OperationDAGLanguageParser.hh.

Referenced by addToken(), showTokenizedSource(), and ~TokenizerData().

◆ root

TokenTreeNode TokenizerData::root
private

Definition at line 708 of file OperationDAGLanguageParser.hh.

Referenced by addToken(), and tokenTree().

The documentation for this class was generated from the following file:
Generated by doxygen 1.9.8