\\\\\\[\\]{}()|='; /** * Construct tokenizer * * Create token array with regular repression matching the respective * token. * * @return void */ public function __construct() { $this->tokens = array( /* // Match tokens which require to be at the start of a line before // matching the actual newlines, because they are the indicator for // line starts. array( 'class' => 'ezcDocumentWikiTitleToken', 'match' => '(\\A\\n(?P=+)' . self::WHITESPACE_CHARS . '+)S' ), array( 'class' => 'ezcDocumentWikiTitleToken', 'match' => '(\\A(?P' . self::WHITESPACE_CHARS . '+(?P=+))\\n)S' ), */ array( 'class' => 'ezcDocumentWikiBulletListItemToken', 'match' => '(\\A\\n(?P[:*#]*\\*)' . self::WHITESPACE_CHARS . '*)S' ), array( 'class' => 'ezcDocumentWikiEnumeratedListItemToken', 'match' => '(\\A\\n(?P[:*#]*#)' . self::WHITESPACE_CHARS . '*)S' ), array( 'class' => 'ezcDocumentWikiDefinitionListItemToken', 'match' => '(\\A\\n(?P[:*#]*:)' . self::WHITESPACE_CHARS . '*)S' ), array( 'class' => 'ezcDocumentWikiLiteralLineToken', 'match' => '(\\A\\n(?P' . self::WHITESPACE_CHARS . '))SUs' ), /* array( 'class' => 'ezcDocumentWikiPageBreakToken', 'match' => '(\\A(?P\n' . self::WHITESPACE_CHARS . '*(?P-{4})' . self::WHITESPACE_CHARS . '*)\\n)S' ), array( 'class' => 'ezcDocumentWikiTableRowToken', 'match' => '(\\A(?P\\n)(?P\\|))S' ), */ // Whitespaces array( 'class' => 'ezcDocumentWikiNewLineToken', 'match' => '(\\A' . self::WHITESPACE_CHARS . '*(?P\\r\\n|\\r|\\n))S' ), array( 'class' => 'ezcDocumentWikiWhitespaceToken', 'match' => '(\\A(?P' . self::WHITESPACE_CHARS . '+))S' ), array( 'class' => 'ezcDocumentWikiEndOfFileToken', 'match' => '(\\A(?P\\x0c))S' ), /* // Escape character array( 'class' => 'ezcDocumentWikiEscapeCharacterToken', 'match' => '(\\A(?P~))S' ), */ // Inline markup array( 'class' => 'ezcDocumentWikiMediawikiEmphasisToken', 'match' => '(\\A(?P\'{2,}))S' ), array( 'class' => 'ezcDocumentWikiTextLineToken', 'match' => '(\\A(?P(?P.+)))SUsi' ), /* array( 'class' => 'ezcDocumentWikiMonospaceToken', 'match' => '(\\A(?P##))S' ), array( 'class' => 'ezcDocumentWikiSuperscriptToken', 'match' => '(\\A(?P\\^\\^))S' ), array( 'class' => 'ezcDocumentWikiSubscriptToken', 'match' => '(\\A(?P,,))S' ), array( 'class' => 'ezcDocumentWikiUnderlineToken', 'match' => '(\\A(?P__))S' ), array( 'class' => 'ezcDocumentWikiInlineLiteralToken', 'match' => '(\\A\\{\\{\\{(?P.+?\\}*)\\}\\}\\})Ss' ), array( 'class' => 'ezcDocumentWikiLineBreakToken', 'match' => '(\\A(?P\\\\\\\\))S' ), array( 'class' => 'ezcDocumentWikiImageStartToken', 'match' => '(\\A(?P\\{\\{))S' ), array( 'class' => 'ezcDocumentWikiImageEndToken', 'match' => '(\\A(?P\\}\\}))S' ), */ array( 'class' => 'ezcDocumentWikiLinkStartToken', 'match' => '(\\A(?P\\[\\[))S' ), array( 'class' => 'ezcDocumentWikiLinkEndToken', 'match' => '(\\A(?P\\]\\]))S' ), /* array( 'class' => 'ezcDocumentWikiTableHeaderToken', 'match' => '(\\A(?P\\|=))S' ), */ array( 'class' => 'ezcDocumentWikiSeparatorToken', 'match' => '(\\A(?P\\|))S' ), /* array( 'class' => 'ezcDocumentWikiInterWikiLinkToken', 'match' => '(\\A(?P([A-Za-z]+):(?:[A-Z][a-z0-9_-]+){2,}))S' ), array( 'class' => 'ezcDocumentWikiInternalLinkToken', 'match' => '(\\A(?P(?:[A-Z][a-z]+){2,}))S' ), array( 'class' => 'ezcDocumentWikiExternalLinkToken', 'match' => '(\\A(?P(?P[a-z]+://\S+?))[,.?!:;"\']?(?:' . self::WHITESPACE_CHARS . '|\\n|\\||]]|\\||$))S' ), // Handle plugins array( 'class' => 'ezcDocumentWikiPluginToken', 'match' => '(\\A<<(?P.*?)>>)Ss' ), */ // Match text except array( 'class' => 'ezcDocumentWikiTextLineToken', 'match' => '(\\A(?P[^' . self::TEXT_END_CHARS . ']+))S' ), ); } /** * Tokenize the given string * * The method tries to tokenize the passed strings and returns an array of * ezcDocumentWikiToken struct on succes, or throws a * ezcDocumentTokenizerException, if something could not be matched by any * token. * * @param string $string * @return array */ public function tokenizeString( $string ) { // Remove all comments, since they are ignored anyways and make some // checks a lot harder. $string = preg_replace( '(' . self::NEW_LINE . '?|\\Z))Ss', '', $string ); return parent::tokenizeString( $string ); } /** * Filter tokens * * Method to filter tokens, after the input string ahs been tokenized. The * filter should extract additional information from tokens, which are not * generally available yet, like the depth of a title depending on the * title markup. * * @param array $tokens * @return array */ protected function filterTokens( array $tokens ) { foreach ( $tokens as $nr => $token ) { switch ( true ) { // Extract the title / indentation level from the tokens // length. case $token instanceof ezcDocumentWikiTitleToken: case $token instanceof ezcDocumentWikiParagraphIndentationToken: $token->level = strlen( trim( $token->content ) ); break; } } return $tokens; } } ?>