* array( * 'Hello', * self:SPACE, * 'world!', * ); * * * Non breaking spaces should not be splitted into multiple words, so there * will be no break applied. * * @param string $string * @return array */ public function tokenize( $string ) { $string = preg_replace( '(\\s+)', ' ', $string ); $words = preg_split( '(( ))', $string, -1, PREG_SPLIT_DELIM_CAPTURE ); // Convert spaces to the marker constant foreach ( $words as $key => $word ) { if ( $word === '' ) { unset( $words[$key] ); } else if ( $word === ' ' ) { $words[$key] = self::SPACE; } } return array_values( $words ); } } ?>