feat(parser): Initial implementation of parser as PHP ext

zephir-lang · Dec 5, 2015 · 98e661d · 98e661d · mruz · Dec 14, 2015
1 parent 2efc33c
commit 98e661d
Show file tree

Hide file tree

Showing 31 changed files with 1,043 additions and 755 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,12 +1,13 @@
 .DS_Store
 .idea/
 parser/lemon
+parser/lemon.exe
 *.dSYM/
-bin/zephir-parser
 bin/zephir55
 parser/parser.c
 parser/scanner.c
 parser/parser.out
+parser/Release
 composer.lock
 vendor/
 build

diff --git a/.gitmodules b/.gitmodules
diff --git a/.travis.yml b/.travis.yml
@@ -32,7 +32,7 @@ install:
 
 before_script:
  - if [[ "$TRAVIS_PHP_VERSION" == "7.0" ]]; then rm unit-tests/Extension/PregmatchTest.php; fi
- - $(phpenv which php) compiler.php generate
+ - $(phpenv which php) -d safe_mode=Off -d enable_dl=On compiler.php generate
  - $(phpenv which php) compiler.php stubs
  - $(phpenv which php) compiler.php api
  - (cd ext; export CFLAGS="-g3 -O1 -fno-delete-null-pointer-checks -Wall -fvisibility=hidden -DZEPHIR_RELEASE=1"; phpize && ./configure --silent --enable-test && make -j2 && make install && phpenv config-add ../unit-tests/ci/test.ini)

diff --git a/Library/Bootstrap.php b/Library/Bootstrap.php
@@ -19,6 +19,7 @@
 
 namespace Zephir;
 
+use Zephir\Compiler;
 use Zephir\Commands\CommandAbstract;
 
 /**

diff --git a/Library/Commands/CommandAbstract.php b/Library/Commands/CommandAbstract.php
@@ -97,6 +97,13 @@ public function execute(Config $config, Logger $logger)
         }
         $backend = new $className();
         $compiler = new Compiler($config, $logger, $backend);
+        if (isset($params['parser-compiled'])) {
+            if ($params['parser-compiled'] !== 'force') {
+                $compiler->parserCompiled = true;
+            } else {
+                $compiler->parserCompiled = 'force';
+            }
+        }
         $command = $this->getCommand();
         $compiler->$command($this);
     }

diff --git a/Library/Commands/CommandApi.php b/Library/Commands/CommandApi.php
@@ -78,8 +78,8 @@ public function execute(Config $config, Logger $logger)
                 } else {
                     throw new Exception("Invalid value for argument '$k'");
                 }
-            } else {
-                throw new Exception("Invalid argument '$k''");
+            } else if (!in_array($k, array('parser-compiled'))) {
+                throw new Exception("Invalid argument '$k' for api command'");
             }
         }
 

diff --git a/Library/Compiler.php b/Library/Compiler.php
@@ -32,6 +32,8 @@ class Compiler
 {
     const VERSION = '0.9.1a-dev';
 
+    public $parserCompiled = false;
+
     /**
      * @var CompilerFile[]
      */
@@ -181,13 +183,97 @@ public function addFunction(FunctionDefinition $func, $statement = null)
         $this->functionDefinitions[$funcName] = $func;
     }
 
+    /**
+     * Compile the parser PHP extension
+     */
+    public function compileParser()
+    {
+        if ($this->parserCompiled && $this->parserCompiled !== 'force') {
+            return extension_loaded('zephir_parser');
+        }
+
+        $extFile = null;
+        $currentDir = realpath(dirname(__FILE__)). DIRECTORY_SEPARATOR  . '..' .DIRECTORY_SEPARATOR . 'parser' . DIRECTORY_SEPARATOR ;
+        if (!extension_loaded('zephir_parser') || $this->parserCompiled === 'force') {
+            $this->logger->output('zephir_parser extension not loaded, compiling it');
+            $oldCwd = getcwd();
+            chdir($currentDir);
+            if (Utils::isWindows()) {
+                echo shell_exec('cd parser && cmd /c build_win32.bat');
+                exec('%PHP_DEVPACK%\\phpize --clean', $output, $exit);
+                if (file_exists('Release')) {
+                    exec('rd /s /q "' . $currentDir . '"Release', $output, $exit);
+                }
+                $this->logger->output('Preparing for parser compilation...');
+                exec('%PHP_DEVPACK%\\phpize', $output, $exit);
+                /* fix until https://github.com/php/php-src/commit/9a3af83ee2aecff25fd4922ef67c1fb4d2af6201 hits all supported PHP builds */
+                $fixMarker = "/* zephir_phpize_fix */";
+                $configureFile = file_get_contents("configure.js");
+                $configureFix = array("var PHP_ANALYZER = 'disabled';", "var PHP_PGO = 'no';", "var PHP_PGI = 'no';");
+                if (strpos($configureFile, $fixMarker) === false) {
+                    file_put_contents("configure.js", $fixMarker . PHP_EOL . implode($configureFix, PHP_EOL) . PHP_EOL . $configureFile);
+                }
+                exec('configure --enable-zephir_parser');
+                $this->logger->output('Compiling the parser...');
+                exec('nmake 2> compile-errors.log 1> compile.log', $output, $exit);
+                echo file_get_contents('compile-errors.log') . PHP_EOL;
+                $buildLog = file_get_contents('compile.log');
+                echo $buildLog . PHP_EOL;
+                $buildType = 'Release';
+                if (strpos($buildLog, 'Debug_TS\\') !== false) {
+                    $buildType = 'Debug_TS';
+                } else if (strpos($buildLog, 'Release_TS\\') !== false) {
+                    $buildType = 'Release_TS';
+                } else if (strpos($buildLog, 'Debug\\') !== false) {
+                    $buildType = 'Debug';
+                }
+                copy($buildType . '/php_zephir_parser.dll', 'php_zephir_parser.dll');
+                $extFile = $currentDir . 'php_zephir_parser.dll';
+            } else {
+                echo shell_exec('cd parser && ./build_linux.sh');
+                exec('make clean && phpize --clean', $output, $exit);
+                $this->logger->output('Preparing for parser compilation...');
+                exec('phpize', $output, $exit);
+
+                exec('export CC="gcc" && ./configure --enable-zephir_parser');
+                $this->logger->output('Compiling the parser...');
+
+                exec(
+                    '(make --silent -j2 2> ./compile-errors.log 1> ./compile.log)',
+                    $output,
+                    $exit
+                );
+                echo file_get_contents('compile-errors.log') . PHP_EOL;
+                echo file_get_contents('compile.log') . PHP_EOL;
+                copy('modules/zephir_parser.so', 'zephir_parser.so');
+                $extFile = $currentDir . 'zephir_parser.so';
+            }
+            chdir($oldCwd);
+            if (!$extFile || !file_exists($extFile)) {
+                throw new Exception('The zephir parser extension could not be found or compiled!');
+            }
+        }
+        return $extFile;
+    }
+
     /**
      * Pre-compiles classes creating a CompilerFile definition
      *
      * @param string $filePath
      */
     protected function preCompile($filePath)
     {
+        $parserExt = $this->compileParser();
+        /* Check if we need to load the parser extension and also allow users to manage the zephir parser extension on their own (zephir won't handle updating)*/
+        if ($parserExt && $parserExt !== true) {
+            $cmd = PHP_BINARY . ' -dextension="'.$parserExt . '" ' . implode(' ', $_SERVER['argv']) . ' --parser-compiled';
+            echo $cmd;
+            passthru($cmd, $exitCode);
+            exit($exitCode);
+        }
+        if (!$parserExt) {
+            throw new Exception('The zephir parser extension is not loaded!');
+        }
         if (preg_match('#\.zep$#', $filePath)) {
             $className = str_replace(DIRECTORY_SEPARATOR, '\\', $filePath);
             $className = preg_replace('#.zep$#', '', $className);

diff --git a/Library/CompilerFile.php b/Library/CompilerFile.php
@@ -156,32 +156,29 @@ public function genIR(Compiler $compiler)
         $compilePath = DIRECTORY_SEPARATOR . Compiler::VERSION . DIRECTORY_SEPARATOR . $normalizedPath . ".js";
         $zepRealPath = realpath($this->_filePath);
 
-        if (PHP_OS == "WINNT") {
-            $zephirParserBinary = ZEPHIRPATH . 'bin\zephir-parser.exe';
-        } else {
-            $zephirParserBinary = ZEPHIRPATH . 'bin/zephir-parser';
-        }
-
-        if (!file_exists($zephirParserBinary)) {
-            throw new Exception($zephirParserBinary . ' was not found');
-        }
-
         $changed = false;
+
         $fileSystem = $compiler->getFileSystem();
         if ($fileSystem->exists($compilePath)) {
             $modificationTime = $fileSystem->modificationTime($compilePath);
-            if ($modificationTime < filemtime($zepRealPath) || $modificationTime < filemtime($zephirParserBinary)) {
-                $fileSystem->system($zephirParserBinary . ' ' . $zepRealPath, 'stdout', $compilePath);
+            if ($modificationTime < filemtime($zepRealPath)) {
                 $changed = true;
             }
         } else {
-            $fileSystem->system($zephirParserBinary . ' ' . $zepRealPath, 'stdout', $compilePath);
             $changed = true;
         }
 
+        $ir = null;
+        if ($changed) {
+            $ir = zephir_parse_file(file_get_contents($zepRealPath), $zepRealPath);
+            $fileSystem->write($compilePath, json_encode($ir, JSON_PRETTY_PRINT));
+        }
+
         if ($changed || !$fileSystem->exists($compilePath . '.php')) {
-            $json = json_decode($fileSystem->read($compilePath), true);
-            $data = '<?php return ' . var_export($json, true) . ';';
+            if (!isset($ir)) {
+                $ir = json_decode($fileSystem->read($compilePath), true);
+            }
+            $data = '<?php return ' . var_export($ir, true) . ';';
             $fileSystem->write($compilePath . '.php', $data);
         }
 

diff --git a/WINDOWS.md b/WINDOWS.md
@@ -1,69 +1,69 @@
-Zephir Installation/Usage Guide (Windows)
-===================
-This guide explains how to use zephir using a windows operating system.  
-Some parts are optional, when you have a specific PHP version.  
-Parts which are only necessary for a specific PHP version, are marked as such.  
-PHP-Version requirements are marked using ``[]``
-
-Software Requirements [PHP 5.5 or later]
------------------------
-- [Install Visual Studio 2012 Express](http://www.microsoft.com/en-US/download/details.aspx?id=34673)
-(You should start it and activate it)
-
-Software Requirements [below PHP 5.5]
------------------------
-- [Install Windows SDK 6.1](http://www.microsoft.com/en-us/download/details.aspx?id=24826)   
-WARNING: This usually takes very long to install and is very big
-- [Install Visual Studio 2008 Express (after SDK 6.1!)](http://go.microsoft.com/fwlink/?LinkId=104679)  
-Install C++ Express Edition, (You should start and activate it)
-
-Software Requirements General
------------------------
-
-- [Install PHP (NTS)](http://windows.php.net/download/) 
-    - Download and extract it 
-    - Make sure it is in the PATH, as for example below:
-    ```
-    setx path "%path%;c:\path-to-php\"
-    ```
-- [Install PHP SDK](http://windows.php.net/downloads/php-sdk/)   
-(Currently "php-sdk-binary-tools-20110915.zip" is the newest)
-```
-setx php_sdk "c:\path-to-php-sdk"
-```
-
-- [Download PHP Developer Pack(NTS!)](http://windows.php.net/downloads/releases/)  
-(or build it yourself with ``--enable-debug --disable-zts`` by using the PHP-SDK)
-```
-setx php_devpack "c:\path-to-extracted-devpack"
-```
-
-Installation of Zephir
-----------------------
-- Clone/Download the repostiory and set the path as below 
-```
-setx path "%path%;c:\path-to-zephir\bin"
-```
-
-Usage of Zephir
-----------------
-- [**PHP5.5 or later**] Open the Visual Studio 2012 Command Prompt  
-(Find it by searching for cmd or just open ``"%VS110COMNTOOLS%\VsDevCmd"``)
-- [**below PHP5.5**] Open the Visual Studio 2008 Command Prompt  
-(Find it by search for cmd or just open ``"%VS90COMNTOOLS%\vsvars32"``)
-- Execute ``%PHP_SDK%\bin\phpsdk_setvars``
-- ``CD`` to your extension and ``zephir build``
-- Take the built ``.dll`` from ``your_ext/Release/php_extname.dll``
-
-Building the parser
---------------------
-- Build zephir/json-c using VS2012 and copy the resulting **json-c.lib** (static library)
-  to the parser directory
-- Requirements: Copy re2c.exe to the parser folder (from PHP-SDK for example)
-- You may have to adjust the paths in buildWin32.bat (if you for example do not use VS2012 on a x64 machine)
-- Run parser/buildWin32.bat to build the parser.exe
-
-
-Additional Links
-------------------
-Building PHP under Windows: https://wiki.php.net/internals/windows/stepbystepbuild
+Zephir Installation/Usage Guide (Windows)
+===================
+This guide explains how to use zephir using a windows operating system.
+Some parts are optional, when you have a specific PHP version.
+Parts which are only necessary for a specific PHP version, are marked as such.
+PHP-Version requirements are marked using ``[]``
+
+Software Requirements [PHP 5.5 or later]
+-----------------------
+- [Install Visual Studio 2012 Express](http://www.microsoft.com/en-US/download/details.aspx?id=34673)
+(You should start it and activate it)
+
+Software Requirements [below PHP 5.5]
+-----------------------
+- [Install Windows SDK 6.1](http://www.microsoft.com/en-us/download/details.aspx?id=24826)
+WARNING: This usually takes very long to install and is very big
+- [Install Visual Studio 2008 Express (after SDK 6.1!)](http://go.microsoft.com/fwlink/?LinkId=104679)
+Install C++ Express Edition, (You should start and activate it)
+
+Software Requirements General
+-----------------------
+
+- [Install PHP (NTS)](http://windows.php.net/download/)
+    - Download and extract it
+    - Make sure it is in the PATH, as for example below:
+    ```
+    setx path "%path%;c:\path-to-php\"
+    ```
+- [Install PHP SDK](http://windows.php.net/downloads/php-sdk/)
+(Currently "php-sdk-binary-tools-20110915.zip" is the newest)
+```
+setx php_sdk "c:\path-to-php-sdk"
+```
+
+- [Download PHP Developer Pack(NTS!)](http://windows.php.net/downloads/releases/)
+(or build it yourself with ``--enable-debug --disable-zts`` and ``nmake build-devel`` or just ``nmake snap`` by using the PHP-SDK)
+```
+setx php_devpack "c:\path-to-extracted-devpack"
+```
+
+Installation of Zephir
+----------------------
+- Clone/Download the repostiory and set the path as below
+```
+setx path "%path%;c:\path-to-zephir\bin"
+```
+
+Usage of Zephir
+----------------
+- [**PHP5.5 or later**] Open the Visual Studio 2012 Command Prompt
+(Find it by searching for cmd or just open ``"%VS110COMNTOOLS%\VsDevCmd"``)
+- [**below PHP5.5**] Open the Visual Studio 2008 Command Prompt
+(Find it by search for cmd or just open ``"%VS90COMNTOOLS%\vsvars32"``)
+- Execute ``%PHP_SDK%\bin\phpsdk_setvars``
+- ``CD`` to your extension and ``zephir build``
+- Take the built ``.dll`` from ``your_ext/Release/php_extname.dll``
+
+Building the parser
+--------------------
+- Build zephir/json-c using VS2012 and copy the resulting **json-c.lib** (static library)
+  to the parser directory
+- Requirements: Copy re2c.exe to the parser folder (from PHP-SDK for example)
+- You may have to adjust the paths in buildWin32.bat (if you for example do not use VS2012 on a x64 machine)
+- Run parser/buildWin32.bat to build the parser.exe
+
+
+Additional Links
+------------------
+Building PHP under Windows: https://wiki.php.net/internals/windows/stepbystepbuild
diff --git a/appveyor.yml b/appveyor.yml
@@ -45,17 +45,12 @@ install:
   - ps: (new-object net.webclient).DownloadFile('http://windows.php.net/downloads/releases/php-devel-pack-' + ${env:PHP_VERSION} + '-Win32-VC' + ${env:PHP_VC} + '-' + ${env:BUILD_PLATFORM} + '.zip', ${env:APPVEYOR_BUILD_FOLDER} + '\..\php-dev.zip')
   - 7z.exe x php-dev.zip | FIND /V "ing  "
   - mv php-%PHP_VERSION%-devel-VC11-%BUILD_PLATFORM% php-devpack
-  - echo Building JSON-C
-  - cd %APPVEYOR_BUILD_FOLDER%
-  - cd json-c
-  - msbuild.exe json-c.sln /t:Build /p:Configuration=Debug;Platform=Win32
   - echo Building Zephir Parser
-  - 'copy Debug\json-c.lib ..\parser\json-c.lib'
-  - 'cd ..\parser'
+  - cd %APPVEYOR_BUILD_FOLDER%
+  - 'cd parser/parser'
   - 'copy %PHP_SDK%\bin\re2c.exe .\re2c.exe'
-  - buildWin32.bat
-  - echo Building PHP [%PHP_VERSION%]
   - '"%VS110COMNTOOLS%\VsDevCmd" %BUILD_PLATFORM%'
+  - echo Building PHP [%PHP_VERSION%]
   - '%PHP_SDK%\bin\phpsdk_setvars'
   - 'cd %APPVEYOR_BUILD_FOLDER%\..\php'
   - 'echo extension_dir=%APPVEYOR_BUILD_FOLDER%\..\php\ext > php.ini'
@@ -66,7 +61,6 @@ install:
   - 'set PATH=%cd%;%PATH%'
   - echo Building Zephir
   - cd %APPVEYOR_BUILD_FOLDER%
-  - 'copy parser\parser.exe bin\zephir-parser.exe'
   - if not exist vendor (php -r "readfile('https://getcomposer.org/installer');" | php & php composer.phar --prefer-source install)
 
 build_script:
@@ -75,5 +69,10 @@ build_script:
   - 'type compile.log'
   - 'php vendor\phpunit\phpunit\phpunit.php --debug'
 
+on_failure :
+  - 'dir'
+  - 'type compile-errors.log'
+  - 'type compile.log'
+
 artifacts:
-  - path: bin\zephir-parser.exe
+  - path: parser\Release\php_zephir_parser.dll
-Original file line number
+Diff line change
@@ Expand Up / @@ -19,6 +19,7 @@ @@
     namespace Zephir;
+    use Zephir\Compiler;
     use Zephir\Commands\CommandAbstract;
     /**
@@ Expand Down @@