I wrote this preprocessor just to add a couple features to Python (pipes mostly.) Is there any way I could improve it?
import tokenize, io, types
PREFIX = """
# ITHON START
import sys as _INTERNAL_SYS
class _INTERNAL_LazyIterable:
__slots__ = ('x','y')
def __init__(self, x, y) -> None:
self.x = iter(x)
self.y = iter(y)
def __iter__(self):
return _INTERNAL_LazyIterator(self)
class _INTERNAL_LazyIterator:
__slots__ = ('iterable',)
def __init__(self, iterable) -> None:
self.iterable = iterable
def __next__(self):
try:
return next(self.iterable.x)
except StopIteration:
try:
return next(self.iterable.y)
except StopIteration:
raise StopIteration
class _INTERNAL_LAZYMERGE:
def __init__(self, val):
self.val = val
def __rrshift__(self, other):
return _INTERNAL_LAZYMERGE(other)
def __lshift__(self, other):
return _INTERNAL_LazyIterable(self.val, other)
class _INTERNAL_LPIPE:
def __init__(self, val):
self.val = val
def __rrshift__(self, other):
return _INTERNAL_LPIPE(other)
def __lshift__(self, other):
return other(self.val)
class _INTERNAL_RPIPE:
def __init__(self, action):
self.action = action
def __rrshift__(self, other):
return _INTERNAL_RPIPE(other)
def __lshift__(self, other):
return self.action(other)
_INTERNAL_lpipe = _INTERNAL_LPIPE(None)
_INTERNAL_rpipe = _INTERNAL_RPIPE(None)
_INTERNAL_lazymerge = _INTERNAL_LAZYMERGE(None)
# ITHON END
"""
def translate(file: io.StringIO):
patched_file = io.StringIO(PREFIX + file.read())
skip_token = False
tokens = list(tokenize.generate_tokens(patched_file.readline)) # Precalculate tokens
print(tokens)
for n, i in enumerate(tokens):
type, name,_,_,_ = i
try:
next_type, next_name,_,_,_ = tokens[n + 1]
except IndexError:
next_type, next_name = (None, None)
print(type, name)
if skip_token:
skip_token = False
continue
if type == tokenize.OP and next_type == tokenize.OP:
# Most likely special operation
if name == "|" and next_name == ">": # left pipe
yield tokenize.OP, ">>"
yield tokenize.NAME, "_INTERNAL_lpipe"
yield tokenize.OP, "<<"
skip_token = True
elif name == "<" and next_name == "|": # right pipe
yield tokenize.OP, ">>"
yield tokenize.NAME, "_INTERNAL_rpipe"
yield tokenize.OP, "<<"
skip_token = True
elif name == ":" and next_name == ":": # lazy merge
yield tokenize.OP, ">>"
yield tokenize.NAME, "_INTERNAL_lazymerge"
yield tokenize.OP, "<<"
skip_token = True
else:
yield type,name
elif type == tokenize.OP:
if name == "<>": # barry is flufl
yield tokenize.OP, "!="
else:
yield type, name
else:
yield type,name
script = """
def x():
print(1)
a = "Hello, World!"
b = print
a |> b
b <| a
for i in [1,2,3] :: [4,5,6] :: [7,8,9]:
print(i)
if a <> b:
print("Barry is FLUFL!")
"Hello, World!" |> print
"""
code = tokenize.untokenize(translate(io.StringIO(script)))
print(code)
exec(code)
Example code output:
# ITHON START
import sys as _INTERNAL_SYS
class _INTERNAL_LazyIterable :
__slots__ =('x','y')
def __init__ (self ,x ,y )->None :
self .x =iter (x )
self .y =iter (y )
def __iter__ (self ):
return _INTERNAL_LazyIterator (self )
class _INTERNAL_LazyIterator :
__slots__ =('iterable',)
def __init__ (self ,iterable )->None :
self .iterable =iterable
def __next__ (self ):
try :
return next (self .iterable .x )
except StopIteration :
try :
return next (self .iterable .y )
except StopIteration :
raise StopIteration
class _INTERNAL_LAZYMERGE :
def __init__ (self ,val ):
self .val =val
def __rrshift__ (self ,other ):
return _INTERNAL_LAZYMERGE (other )
def __lshift__ (self ,other ):
return _INTERNAL_LazyIterable (self .val ,other )
class _INTERNAL_LPIPE :
def __init__ (self ,val ):
self .val =val
def __rrshift__ (self ,other ):
return _INTERNAL_LPIPE (other )
def __lshift__ (self ,other ):
return other (self .val )
class _INTERNAL_RPIPE :
def __init__ (self ,action ):
self .action =action
def __rrshift__ (self ,other ):
return _INTERNAL_RPIPE (other )
def __lshift__ (self ,other ):
return self .action (other )
_INTERNAL_lpipe =_INTERNAL_LPIPE (None )
_INTERNAL_rpipe =_INTERNAL_RPIPE (None )
_INTERNAL_lazymerge =_INTERNAL_LAZYMERGE (None )
# ITHON END
def x ():
print (1 )
a ="Hello, World!"
b =print
a >>_INTERNAL_lpipe <<b
b >>_INTERNAL_rpipe <<a
for i in [1 ,2 ,3 ]>>_INTERNAL_lazymerge <<[4 ,5 ,6 ]>>_INTERNAL_lazymerge <<[7 ,8 ,9 ]:
print (i )
if a !=b :
print ("Barry is FLUFL!")
"Hello, World!">>_INTERNAL_lpipe <<print