some helper functions

def make_ascii(str,debug=False):

    s = ""

    for num,i in enumerate(str):

        try:

            i.encode("utf-8")

        except (UnicodeEncodeError, UnicodeDecodeError):

            if debug:

                import sys

                sys.stderr.write("error converting char %d\n" %num )

            continue

        else:

            s += i

    return s

def yield_tokens(line):

    for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)):

        if i % 2:

            if part.startswith('!'):

                yield 'RAW', part[1:]

            else: yield 'CMD', part

        else: yield 'TXT', part

Posted via email from sente's posterous