Generation: Support Python 3 (#2288)

* Generation: Use Python 3-compatible imports.

* Generation: Use range() instead of xrange().

* Generation: Use list comprehensions instead of map().

* Generation: Explicitly use bytes in the Test class.

* Generation: Run unit tests on Python 3 as well.

.circleci/config.yml

@@ -62,15 +62,21 @@ jobs:
       # - run:
       #     name: "Run deploy"
       #     command: ./tools/scripts/deploy.sh
-  "Project lint unit tests on Python 3":
+  "Project lint and generation unit tests on Python 3":
     docker:
       - image: circleci/python:3.7.4
     working_directory: ~/test262
     steps:
       - checkout
+      - run:
+          name: "Install requirements for generation tool"
+          command: python -m pip install --user --requirement tools/generation/requirements.txt
       - run:
           name: "Install requirements for linter tool"
           command: python -m pip install --user --requirement tools/lint/requirements.txt
+      - run:
+          name: "Test the generation tool"
+          command: ./tools/generation/test/run.py
       - run:
           name: "Test the lint tool"
           command: ./tools/lint/test/run.py
@@ -137,7 +143,7 @@ workflows:
   Tools:
     jobs:
       - "Project lint, generation tests and build"
-      - "Project lint unit tests on Python 3"
+      - "Project lint and generation unit tests on Python 3"
   Tests execution:
     jobs:
       - "ChakraCore: New or modified tests execution"

tools/generation/lib/case.py

@@ -4,8 +4,8 @@
 import codecs
 import re
 
-from util.find_comments import find_comments
-from util.parse_yaml import parse_yaml
+from .util.find_comments import find_comments
+from .util.parse_yaml import parse_yaml
 
 regionStartPattern = re.compile(r'-\s+(\S+)')
 

tools/generation/lib/expander.py

@@ -3,8 +3,8 @@
 
 import re, os
 
-from case import Case
-from template import Template
+from .case import Case
+from .template import Template
 
 caseFilenamePattern = re.compile(r'^[^\.].*\.case$')
 templateFilenamePattern = re.compile(r'^[^\.].*\.template$')

tools/generation/lib/template.py

@@ -5,9 +5,9 @@ import os, re
 import codecs, yaml
 from collections import OrderedDict
 
-from util.find_comments import find_comments
-from util.parse_yaml import parse_yaml
-from test import Test
+from .util.find_comments import find_comments
+from .util.parse_yaml import parse_yaml
+from .test import Test
 
 indentPattern = re.compile(r'^(\s*)')
 interpolatePattern = re.compile(r'\{\s*(\S+)\s*\}')

tools/generation/lib/test.py

@@ -3,8 +3,8 @@
 
 import os, re
 
-from util.find_comments import find_comments
-from util.parse_yaml import parse_yaml
+from .util.find_comments import find_comments
+from .util.parse_yaml import parse_yaml
 
 class Test:
     """Representation of a generated test. Specifies a file location which may
@@ -19,7 +19,7 @@ class Test:
 
     def load(self, prefix = None):
         location = os.path.join(prefix or '', self.file_name)
-        with open(location) as handle:
+        with open(location, 'rb') as handle:
             self.source = handle.read()
         self._parse()
 
@@ -59,5 +59,5 @@ class Test:
             else:
                 raise Exception('Directory does not exist: ' + path)
 
-        with open(location, 'w') as handle:
+        with open(location, 'wb') as handle:
             handle.write(self.source)

tools/generation/lib/util/find_comments.py

@@ -23,7 +23,7 @@ def find_comments(source):
     comment = ''
     lineno = 0
 
-    for idx in xrange(len(source)):
+    for idx in range(len(source)):
         if source[idx] == '\n':
             lineno += 1
 

tools/generation/test/run.py

@@ -36,8 +36,8 @@ class TestGeneration(unittest.TestCase):
         actualFiles = self.getFiles(actualPath)
 
         self.assertListEqual(
-            map(lambda x: os.path.relpath(x, expectedPath), expectedFiles),
-            map(lambda x: os.path.relpath(x, actualPath), actualFiles))
+            [os.path.relpath(x, expectedPath) for x in expectedFiles],
+            [os.path.relpath(x, actualPath) for x in actualFiles])
 
         for expectedFile, actualFile in zip(expectedFiles, actualFiles):
             with open(expectedFile) as expectedHandle: