issue #203 Removing LLVM support, see llvm branch for archived version

cmake
Dibyendu Majumdar 3 years ago
parent 05f365352c
commit 66511033fd

@ -3,6 +3,7 @@ os:
- linux
arch:
- amd64
- arm64
compiler:
- gcc
cache: ccache
@ -10,16 +11,9 @@ dist: bionic
addons:
apt:
packages:
- g++
- gcc
- ccache
install:
- curl https://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz | tar -xJf -
script:
- mkdir $TRAVIS_BUILD_DIR/build
- cd $TRAVIS_BUILD_DIR/build && cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_DIR=$TRAVIS_BUILD_DIR/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04/lib/cmake/llvm -G "Unix Makefiles" -DLLVM_JIT=ON ..
- cd $TRAVIS_BUILD_DIR/build && make
- cd $TRAVIS_BUILD_DIR/tests && sh ./run_tests.sh $TRAVIS_BUILD_DIR/build/ravi
- mkdir $TRAVIS_BUILD_DIR/buildmir
- cd $TRAVIS_BUILD_DIR/buildmir && cmake -DCMAKE_BUILD_TYPE=Debug -DLTESTS=ON -DMIR_JIT=ON -G "Unix Makefiles" ..
- cd $TRAVIS_BUILD_DIR/buildmir && make

@ -1,12 +1,9 @@
cmake_minimum_required(VERSION 3.12)
project(Ravi VERSION 1.0.4 LANGUAGES C CXX)
project(Ravi VERSION 1.0.4 LANGUAGES C)
# By default on platforms where MIR is supported (right now Linux X86_64)
# MIR JIT backend is automatically enabled. To disable user must specify
# By default MIR JIT backend is automatically enabled. To disable user must specify
# NO_JIT=ON
# LLVM has to be explicitly specified using LLVM_JIT=ON
option(LLVM_JIT "Controls whether LLVM JIT compilation will be enabled, default is OFF" OFF)
option(NO_JIT "Controls whether JIT should be disabled, default is OFF" OFF)
option(STATIC_BUILD "Build static version of Ravi, default is OFF" OFF)
option(COMPUTED_GOTO "Controls whether the interpreter switch will use computed gotos on gcc/clang, default is ON" ON)
@ -16,14 +13,12 @@ option(RAVICOMP "Controls whether to link in RaviComp" OFF)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
# By default on non-Windows platforms we enable MIR JIT
if (NOT LLVM_JIT
AND NOT NO_JIT)
# By we enable MIR JIT
if (NOT NO_JIT)
set(MIR_JIT ON)
endif ()
if (MIR_JIT)
set(LLVM_JIT OFF)
set(STATIC_BUILD OFF) # Because we need to expose the symbols in the library
endif ()
@ -51,12 +46,6 @@ set(LUA_CORE_SRCS src/lapi.c src/lcode.c src/lctype.c src/ldebug.c src/ldo.c src
set(LUA_LIB_SRCS src/lauxlib.c src/lbaselib.c src/lbitlib.c src/lcorolib.c src/ldblib.c src/liolib.c
src/lmathlib.c src/loslib.c src/ltablib.c src/lstrlib.c src/loadlib.c src/linit.c src/lutf8lib.c)
set(LUA_HEADERS include/lua.h include/luaconf.h include/lualib.h include/lauxlib.h)
# LLVM code gen
set(LLVM_JIT_SRCS src/ravi_llvmjit.cpp src/ravi_llvmtypes.cpp
src/ravi_llvmcodegen.cpp src/ravi_llvmforprep.cpp src/ravi_llvmcomp.cpp
src/ravi_llvmreturn.cpp src/ravi_llvmload.cpp src/ravi_llvmforloop.cpp
src/ravi_llvmarith1.cpp src/ravi_llvmcall.cpp src/ravi_llvmtable.cpp
src/ravi_llvmarith2.cpp src/ravi_llvmtforcall.cpp src/ravi_llvmrest.cpp)
set(MIR_HEADERS mir/mir.h mir/mir-gen.h mir/mir-varr.h mir/mir-dlist.h mir/mir-htab.h
mir/mir-hash.h mir/mir-bitmap.h)
set(MIR_SRCS mir/mir.c mir/mir-gen.c)
@ -89,56 +78,26 @@ if (ASAN)
endif ()
endif ()
if (LLVM_JIT)
find_package(LLVM REQUIRED CONFIG)
message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
message(STATUS "LLVM Definitions ${LLVM_DEFINITIONS}")
message(STATUS "LLVMJIT enabled")
set(JIT_SRCS ${LLVM_JIT_SRCS})
if (MIR_JIT)
message(STATUS "MIRJIT enabled")
set(JIT_SRCS ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS})
set(ALL_MIR_SRCS ${MIR_SRCS} ${C2MIR_SRCS})
if (NOT MSVC)
set_source_files_properties(${LLVM_JIT_SRCS} PROPERTIES
COMPILE_FLAGS "-fno-rtti -fno-exceptions ${LLVM_DEFINITIONS}")
set_source_files_properties(${ALL_MIR_SRCS} PROPERTIES
COMPILE_FLAGS "-fsigned-char")
endif ()
set_property(
SOURCE ${LLVM_JIT_SRCS}
set_property(SOURCE ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS}
APPEND
PROPERTY INCLUDE_DIRECTORIES ${LLVM_INCLUDE_DIRS}
)
# FIXME get rid of this dependency
set_property(
SOURCE ${LUA_CMD_SRCS}
PROPERTY INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/mir;${CMAKE_SOURCE_DIR}/mir/c2mir")
set_property(SOURCE ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS}
APPEND
PROPERTY INCLUDE_DIRECTORIES ${LLVM_INCLUDE_DIRS})
PROPERTY COMPILE_DEFINITIONS "MIR_NO_IO=0;MIR_NO_SCAN=1;MIR_NO_INTERP=1")
if ($ENV{CLION_IDE})
# CLion seems unable to handle include paths set on sources
include_directories(${LLVM_INCLUDE_DIRS})
include_directories("${CMAKE_SOURCE_DIR}/mir;${CMAKE_SOURCE_DIR}/mir/c2mir")
endif ()
else ()
if (MIR_JIT)
message(STATUS "MIRJIT enabled")
set(JIT_SRCS ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS})
set(ALL_MIR_SRCS ${MIR_SRCS} ${C2MIR_SRCS})
if (NOT MSVC)
set_source_files_properties(${ALL_MIR_SRCS} PROPERTIES
COMPILE_FLAGS "-fsigned-char")
endif ()
set_property(SOURCE ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS}
APPEND
PROPERTY INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/mir;${CMAKE_SOURCE_DIR}/mir/c2mir")
set_property(SOURCE ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS}
APPEND
PROPERTY COMPILE_DEFINITIONS "MIR_NO_IO=0;MIR_NO_SCAN=1;MIR_NO_INTERP=1")
if ($ENV{CLION_IDE})
# CLion seems unable to handle include paths set on sources
include_directories("${CMAKE_SOURCE_DIR}/mir;${CMAKE_SOURCE_DIR}/mir/c2mir")
endif ()
else ()
set(JIT_SRCS ${NO_JIT_SRCS})
endif ()
set(JIT_SRCS ${NO_JIT_SRCS})
endif ()
if (RAVICOMP)
@ -177,64 +136,6 @@ elseif (NOT WIN32)
set(EXTRA_LIBRARIES m dl readline)
endif ()
# Enable minimal required LLVM components so that the
# the size of the resulting binary is manageable
if (LLVM_JIT)
if (${LLVM_PACKAGE_VERSION} VERSION_LESS "3.8")
set(LLVM_EXTRA_LIBS ipa)
endif ()
if (NOT ${LLVM_PACKAGE_VERSION} VERSION_LESS "5.0.0")
set(LLVM_EXTRA_LIBS orcjit)
endif ()
message(STATUS "SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}")
if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
set(LLVM_LIBS_PROCESSOR
X86CodeGen
X86AsmParser
X86Disassembler
X86Desc
X86Info
X86Utils
)
if (${LLVM_PACKAGE_VERSION} VERSION_LESS "9.0.0")
list(APPEND LLVM_LIBS_PROCESSOR X86AsmPrinter)
endif ()
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
set(LLVM_LIBS_PROCESSOR
ARMCodeGen
ARMAsmParser
ARMDisassembler
ARMAsmPrinter
ARMDesc
ARMInfo
)
endif ()
llvm_map_components_to_libnames(LLVM_LIBS
Analysis
AsmParser
AsmPrinter
BitReader
Core
CodeGen
ExecutionEngine
InstCombine
${LLVM_EXTRA_LIBS}
ipo
MC
MCJIT
MCParser
Object
RuntimeDyld
ScalarOpts
Support
Target
TransformUtils
${LLVM_LIBS_PROCESSOR}
)
message(STATUS "LLVM_LIBS ${LLVM_LIBS}")
endif ()
set(LIBRAVI_NAME libravi)
#Main library
@ -244,7 +145,7 @@ add_library(${LIBRAVI_NAME} ${LIBRAVI_BUILD_TYPE}
${LUA_CORE_SRCS}
${JIT_SRCS}
${ADDON_SRCS})
target_link_libraries(${LIBRAVI_NAME} ${EXTRA_LIBRARIES} ${LLVM_LIBS} ${MIRJIT_LIBRARIES} ${RAVICOMP_LIBRARIES})
target_link_libraries(${LIBRAVI_NAME} ${EXTRA_LIBRARIES} ${MIRJIT_LIBRARIES} ${RAVICOMP_LIBRARIES})
# Main Ravi executable
add_executable(ravi ${LUA_CMD_SRCS})
@ -281,13 +182,7 @@ if (NOT LTESTS)
PROPERTY COMPILE_DEFINITIONS NO_LUA_DEBUG)
set(NO_LUA_DEBUG 1)
endif ()
if (LLVM_JIT)
set_property(
TARGET ${LIBRAVI_NAME} ravi
APPEND
PROPERTY COMPILE_DEFINITIONS "USE_LLVM=1")
set(USE_LLVM 1)
elseif (MIR_JIT)
if (MIR_JIT)
set_property(
TARGET ${LIBRAVI_NAME} ravi
APPEND
@ -325,8 +220,6 @@ elseif (UNIX)
APPEND
PROPERTY COMPILE_DEFINITIONS "LUA_USE_LINUX=1")
endif ()
set_property(TARGET ${LIBRAVI_NAME} PROPERTY CXX_STANDARD 14)
set_property(TARGET ${LIBRAVI_NAME} PROPERTY CXX_EXTENSIONS OFF)
include(GNUInstallDirs)
configure_file(ravi-config.h.in ravi-config.h @ONLY)

@ -5,7 +5,7 @@ Ravi Programming Language
:target: https://travis-ci.org/dibyendumajumdar/ravi
Ravi is a dialect of `Lua <http://www.lua.org/>`_ with limited optional static typing and
features `MIR <https://github.com/vnmakarov/mir>`_ and `LLVM <http://www.llvm.org/>`_ powered JIT compilers.
features `MIR <https://github.com/vnmakarov/mir>`_ powered JIT compilers.
The name Ravi comes from the Sanskrit word for the Sun.
Interestingly a precursor to Lua was `Sol <http://www.lua.org/history.html>`_ which had support for
static types; Sol means the Sun in Portugese.
@ -35,8 +35,7 @@ Features
* Compatibility with Lua 5.3 (see Compatibility section below)
* Generational GC from Lua 5.4
* ``defer`` statement for releasing resources
* Compact JIT backend `MIR <https://github.com/vnmakarov/mir>`_; only Linux and x86-64 supported for now.
* `LLVM <http://www.llvm.org/>`_ supported as alternative JIT backend.
* Compact JIT backend `MIR <https://github.com/vnmakarov/mir>`_.
* A `distribution with batteries <https://github.com/dibyendumajumdar/Suravi>`_.
* A `Visual Studio Code debugger extension <https://marketplace.visualstudio.com/items?itemName=ravilang.ravi-debug>`_ - interpreted mode debugger.
@ -44,7 +43,6 @@ Documentation
=============
* For the Lua extensions in Ravi see the `Reference Manual <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-reference.html>`_.
* `MIR JIT Build instructions <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-mir-instructions.html>`_.
* `LLVM JIT Build instructions <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-llvm-instructions.html>`_.
* Also see `Ravi Documentation <http://the-ravi-programming-language.readthedocs.org/en/latest/index.html>`_.
* and the slides I presented at the `Lua 2015 Workshop <http://www.lua.org/wshop15.html>`_.
@ -99,6 +97,7 @@ When JIT compilation is enabled there are following additional constraints:
* Ravi will only execute JITed code from the main Lua thread; any secondary threads (coroutines) execute in interpreter mode.
* In JITed code tailcalls are implemented as regular calls so unlike the interpreter VM which supports infinite tail recursion JIT compiled code only supports tail recursion to a depth of about 110 (issue #17)
* Debug api and hooks are not supported in JIT mode
History
=======
@ -115,11 +114,13 @@ History
- Created `Ravi with batteries <https://github.com/dibyendumajumdar/Suravi>`_.
* 2019
- New language feature - `defer` statement
- New JIT backend `MIR <https://github.com/vnmakarov/mir>`_.
* 2020 (Plan)
- `New optimizing byte code generator based on new parser / type checker <https://github.com/dibyendumajumdar/ravi-compiler>`_
- New JIT backend `MIR <https://github.com/vnmakarov/mir>`_.
* 2020
- `New parser / type checker / compiler <https://github.com/dibyendumajumdar/ravi-compiler>`_
- Generational GC back-ported from Lua 5.4
- Support for `LLVM backend <https://github.com/dibyendumajumdar/ravi/tree/llvm>`_ archived
* 2021 (Plan)
- Integrated AOT and JIT compilation support
- Ravi 1.0 release
License

@ -5,4 +5,4 @@ The scripts here are unsupported - these are just my personal
build scripts.
The unix LLVM debug builds enable 'ltests' and address sanitizer.
The debug builds enable 'ltests' and address sanitizer.

@ -1,16 +0,0 @@
# Run this on LLVM 10 source dir
mkdir build
cd build
cmake3 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/Software/llvm10 \
-DLLVM_TARGETS_TO_BUILD="X86" \
-DLLVM_BUILD_TOOLS=OFF \
-DLLVM_INCLUDE_TOOLS=OFF \
-DLLVM_BUILD_EXAMPLES=OFF \
-DLLVM_INCLUDE_EXAMPLES=OFF \
-DLLVM_BUILD_TESTS=OFF \
-DLLVM_INCLUDE_TESTS=OFF \
-DLLVM_OPTIMIZED_TABLEGEN=ON \
..
make install

@ -1,5 +0,0 @@
mkdir llvm32
cd llvm32
rem cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM37_32\share\llvm\cmake -DBUILD_STATIC=OFF ..
cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM39_32\lib\cmake\llvm -DSTATIC_BUILD=OFF ..
cd ..

@ -1,5 +0,0 @@
mkdir llvm32
cd llvm32
rem cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM37_32\share\llvm\cmake ..
cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM39\lib\cmake\llvm -DSTATIC_BUILD=ON ..
cd ..

@ -1,5 +0,0 @@
mkdir llvm32d
cd llvm32d
rem cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM37_32\share\llvm\cmake ..
cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM39D_32\lib\cmake\llvm -DSTATIC_BUILD=ON ..
cd ..

@ -1,6 +0,0 @@
mkdir llvm64d
cd llvm64d
rem cmake -DCMAKE_INSTALL_PREFIX=c:\ravi64llvmd -G "Visual Studio 14 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\LLVM37debug\share\llvm\cmake ..
rem cmake -DSTATIC_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=c:\d\ravi64llvmd -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DEMBEDDED_DMRC=ON -DLLVM_DIR=c:\d\LLVM39D64\lib\cmake\llvm ..
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm501d\lib\cmake\llvm ..
cd ..

@ -1,7 +0,0 @@
mkdir llvm64
cd llvm64
rem pre LLVM 3.9
rem cmake -DCMAKE_INSTALL_PREFIX=c:\ravi -G "Visual Studio 14 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\LLVM37\share\llvm\cmake ..
rem cmake -DCMAKE_INSTALL_PREFIX=c:\ravi -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\d\LLVM40_64\lib\cmake\llvm ..
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DSTATIC_BUILD=ON -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm601r\lib\cmake\llvm ..
cd ..

@ -1,5 +0,0 @@
rmdir /s llvm8
mkdir llvm8
cd llvm8
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Release -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm801\lib\cmake\llvm ..
cd ..

@ -1,5 +0,0 @@
rmdir /s llvm10d
mkdir llvm10d
cd llvm10d
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 16 2019" -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm10d\lib\cmake\llvm ..
cd ..

@ -1,6 +0,0 @@
rmdir /s llvm9r
mkdir llvm9r
cd llvm9r
rem cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm900r\lib\cmake\llvm ..
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 16 2019" -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm900r\lib\cmake\llvm ..
cd ..

@ -1,4 +0,0 @@
mkdir omrjit
cd omrjit
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Debug -DOMR_JIT=ON ..
cd ..

@ -1,4 +0,0 @@
mkdir xcodellvm
cd xcodellvm
#cmake -DCMAKE_BUILD_TYPE=Debug -G Xcode -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM/share/llvm/cmake ..
cmake -DCMAKE_BUILD_TYPE=Debug -G Xcode -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM5/lib/cmake/llvm ..

@ -1,5 +0,0 @@
mkdir buildllvmd
cd buildllvmd
#cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM/share/llvm/cmake ..
#cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DLTESTS=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM5/lib/cmake/llvm ..
cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DLTESTS=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravillvm -DLLVM_DIR=$HOME/Software/llvm600/lib/cmake/llvm ..

@ -1,7 +0,0 @@
rm -rf buildllvm
mkdir buildllvm
cd buildllvm
#cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM/share/llvm/cmake ..
#cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM5/lib/cmake/llvm ..
#cmake -DCMAKE_BUILD_TYPE=Release -DSTATIC_BUILD=ON -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/Software/ravi -DLLVM_DIR=$HOME/Software/llvm801/lib/cmake/llvm ..
cmake3 -DCMAKE_BUILD_TYPE=Release -DSTATIC_BUILD=ON -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/Software/ravi -DLLVM_DIR=$HOME/Software/llvm10/lib/cmake/llvm ..

@ -1,3 +0,0 @@
mkdir omrjit
cd omrjit
cmake -DCMAKE_BUILD_TYPE=Release -DOMR_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi ..

@ -12,20 +12,15 @@ RUN set -x \
&& cd /Software \
&& tar xvf "cmake-3.14.5-Linux-x86_64.tar.gz" \
&& rm -rf "/Software/cmake-3.14.5-Linux-x86_64.tar.gz" \
&& wget -O "/Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" "http://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" \
&& cd /Software \
&& tar xvf "clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" \
&& rm -rf "/Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" \
&& mkdir -p /sources \
&& cd /sources \
&& git clone https://github.com/dibyendumajumdar/ravi.git \
&& cd /sources/ravi \
&& mkdir build \
&& cd build \
&& /Software/cmake-3.14.5-Linux-x86_64/bin/cmake -DSTATIC_BUILD=ON -DCMAKE_INSTALL_PREFIX=/Software/ravi -DLLVM_JIT=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_DIR=/Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04/lib/cmake/llvm .. \
&& /Software/cmake-3.14.5-Linux-x86_64/bin/cmake -DCMAKE_INSTALL_PREFIX=/Software/ravi -DCMAKE_BUILD_TYPE=Release .. \
&& make install \
&& rm -rf /Software/cmake-3.14.5-Linux-x86_64 \
&& rm -rf /Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04 \
&& rm -rf /sources \
&& apt-get autoremove \
&& apt-get remove -y --purge git wget build-essential \

@ -1,121 +0,0 @@
/******************************************************************************
* Copyright (C) 2015 Dibyendu Majumdar
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
******************************************************************************/
#ifndef RAVI_LLVM_H
#define RAVI_LLVM_H
#ifdef USE_LLVM
#include "llvm/Config/llvm-config.h"
#if (LLVM_VERSION_MAJOR < 3 || LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR < 5 || LLVM_VERSION_MAJOR == 7)
#error Unsupported LLVM version
#endif
#if LLVM_VERSION_MAJOR >= 5
#define USE_ORC_JIT 1
#else
#define USE_ORC_JIT 0
#endif
#if LLVM_VERSION_MAJOR >= 8 && !defined(_WIN32)
#define USE_ORCv2_JIT 0
#else
#define USE_ORCv2_JIT 0
#endif
#if LLVM_VERSION_MAJOR >= 10
#undef USE_ORCv2_JIT
#define USE_ORCv2_JIT 1
#endif
// In lua.c we include this just to get version numbers
// We cannot have C++ headers in that case
#ifdef __cplusplus
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IR/Metadata.h"
#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR < 7
#include "llvm/PassManager.h"
#else
#include "llvm/IR/LegacyPassManager.h"
#endif
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/DynamicLibrary.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Support/FormattedStream.h"
#if USE_ORC_JIT || USE_ORCv2_JIT
#include "llvm/ADT/STLExtras.h"
#include "llvm/ExecutionEngine/JITSymbol.h"
#include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
#include "llvm/IR/Mangler.h"
#include "llvm/Support/Error.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Scalar/GVN.h"
#if LLVM_VERSION_MAJOR >= 8
#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
#include "llvm/ExecutionEngine/Orc/Legacy.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#endif
#endif
#include <algorithm>
#include <cassert>
#include <cstdlib>
#include <memory>
#include <string>
#include <cstdio>
#include <vector>
#endif //__cplusplus
#endif //USE_LLVM
#endif

File diff suppressed because it is too large Load Diff

@ -14,16 +14,11 @@ Contents:
ravi-overview
ravi-reference
ravi-mir-instructions
ravi-llvm-instructions
lua-introduction
lua_bytecode_reference
lua-parser
ravi-internals
ravi-jit-compilation-hook
ravi-lua-types
llvm-tbaa
ravi-benchmarks
ravi-jit-status
Indices and tables

@ -1,169 +0,0 @@
LLVM Notes
==========
The notes below apply to LLVM 3.5.1 unless noted otherwise. All reflect my understanding - so if anything here is incorrect please log an issue and I will correct.
Structs and Unions
------------------
LLVM does not support defining union types so we need to basically use a ``struct`` of appropriate size and cast it as we need. The main thing to be careful about is to ensure that the ``struct`` is of the same size as the ``union``.
An example is::
///*
//** Information about a call.
//** When a thread yields, 'func' is adjusted to pretend that the
//** top function has only the yielded values in its stack; in that
//** case, the actual 'func' value is saved in field 'extra'.
//** When a function calls another with a continuation, 'extra' keeps
//** the function index so that, in case of errors, the continuation
//** function can be called with the correct top.
//*/
// typedef struct CallInfo {
// StkId func; /* function index in the stack */
// StkId top; /* top for this function */
// struct CallInfo *previous, *next; /* dynamic call link */
// union {
// struct { /* only for Lua functions */
// StkId base; /* base for this function */
// const Instruction *savedpc;
// } l;
// struct { /* only for C functions */
// lua_KFunction k; /* continuation in case of yields */
// ptrdiff_t old_errfunc;
// lua_KContext ctx; /* context info. in case of yields */
// } c;
// } u;
// ptrdiff_t extra;
// short nresults; /* expected number of results from this function */
// lu_byte callstatus;
//} CallInfo;
Above the union ``u`` has two members of unequal size. To handle this I created the following two sub-types of equal size - note the extra dummy field in the first type::
elements.clear();
elements.push_back(StkIdT); /* base */
elements.push_back(pInstructionT); /* savedpc */
elements.push_back(
C_ptrdiff_t); /* dummy to make this same size as the other member */
CallInfo_lT = llvm::StructType::create(elements);
elements.clear();
elements.push_back(plua_KFunctionT); /* k */
elements.push_back(C_ptrdiff_t); /* old_errfunc */
elements.push_back(lua_KContextT); /* ctx */
CallInfo_cT = llvm::StructType::create(elements);
Then as I intend to use the ``u.l`` field more often, I used the following definition for ``CallInfo``::
CallInfoT = llvm::StructType::create(context, "ravi.CallInfo");
pCallInfoT = llvm::PointerType::get(CallInfoT, 0);
elements.clear();
elements.push_back(StkIdT); /* func */
elements.push_back(StkIdT); /* top */
elements.push_back(pCallInfoT); /* previous */
elements.push_back(pCallInfoT); /* next */
elements.push_back(
CallInfo_lT); /* u.l - as we will typically access the lua call details
*/
elements.push_back(C_ptrdiff_t); /* extra */
elements.push_back(llvm::Type::getInt16Ty(context)); /* nresults */
elements.push_back(lu_byteT); /* callstatus */
CallInfoT->setBody(elements);
JIT Compilation Error on Windows
--------------------------------
Note: The latest LLVM version from LLVM source repository appears to support COEFF format. So below
applies to version 3.6.x and 3.5.x.
On Windows when we attempt to JIT compile we get an error saying incompatible object format
Reading posts on mailing lists I found that the issue is that COEFF
format is not supported and therefore we need to set -elf as the object
format::
#include "llvm/Support/Host.h"
/* some code */
#ifdef _WIN32
auto triple = llvm::sys::getProcessTriple();
module->setTargetTriple(triple + "-elf");
#endif
Memory Management
-----------------
It appears that most things in LLVM are owned by the parent object and when the parent object is deleted the children go too. So in my code the main objects I delete are the ``ExecutionEngine`` and ``Module``. Once a module is associated with an engine then only the engine needs to be explicitly deleted is my understanding.
It doesn't help that the tutorial available does not attempt to delete objects / release memory!
MCJIT Engines, Modules and Functions
------------------------------------
Functions live inside Modules but once a Module is finalized (compiled) then no further functions can be added to it. Although an ``MCJIT`` instance (engine) can support multiples modules, the recommendation is to ensure each module is assigned its own engine. The rationale for this is not explained.
Struct Assign
-------------
My understanding is that to perform assignment of a struct value, one must call the intrinsic ``memcpy`` function. Example of code that does this::
llvm::Value *src;
llvm::Value *dest;
// First get the declaration for the inttrinsic memcpy
llvm::SmallVector<llvm::Type *, 3> vec;
vec.push_back(def->types->C_pcharT); /* i8 */
vec.push_back(def->types->C_pcharT); /* i8 */
vec.push_back(def->types->C_intT);
llvm::Function *f = llvm::Intrinsic::getDeclaration(
def->raviF->module(), llvm::Intrinsic::memcpy, vec);
lua_assert(f);
// Cast src and dest to i8*
llvm::Value *dest_ptr =
def->builder->CreateBitCast(dest, def->types->C_pcharT);
llvm::Value *src_ptr = def->builder->CreateBitCast(src, def->types->C_pcharT);
// Create call to intrinsic memcpy
values_.clear();
values_.push_back(dest_ptr);
values_.push_back(src_ptr);
values_.push_back(llvm::ConstantInt::get(def->types->C_intT, sizeof(TValue)));
values_.push_back(
llvm::ConstantInt::get(def->types->C_intT, sizeof(L_Umaxalign)));
values_.push_back(def->types->kFalse);
def->builder->CreateCall(f, values_);
Note that the call to memcpy supply an alignment.
Accessing ``extern`` functions from JIT compiled code
-----------------------------------------------------
If the JITed function needs to access ``extern`` functions that are statically linked and not exported as dynamic symbols (e.g. in Visual C++) then we need some extra steps.
From reading posts on the subject it appears that the way to do this is to add a global mapping in the ``ExecutionEngine`` by calling the
``addGlobalMapping()`` method. However this doesn't work with MCJIT due to a bug! So we need to use a workaround. Apparently there are two
solutions:
* Create a custom memory manager that resolves the ``extern`` functions.
* Add the symbol to the global symbols by calling ``llvm::sys::DynamicLibrary::AddSymbol()``.
I am using the latter approach for now.
GEP instruction
---------------
The GEP instruction cannot compute addresses of fields in a pointer member - as the pointer needs to be 'loaded' first. This is explained in the `GEP FAQ <http://llvm.org/docs/GetElementPtr.html#id6>`_.
Hooking up Optimization Passes
------------------------------
The LLVM documentation does not provide guidance on how the optimization passes should be hooked up. There are descriptions of what the passes do, but if you are new to LLVM and trying to work out which passes to use and in what order, then there is not much help available. The `Kaleidoscope Sample <http://www.llvm.org/docs/tutorial/LangImpl4.html>`_ shows a small example of how optimization passes may be hooked up.
Fortunately it seems that there is a `PassManagerBuilder <http://llvm.org/docs/doxygen/html/classllvm_1_1PassManagerBuilder.html>`_ component that allows easy setup of the standard passes for a C like language. Unfortunately there isn't much guidance on how to use this either. The best source of information I found was an example toy compiler by `David Chisnall <http://cs.swan.ac.uk/~csdavec/FOSDEM12/compiler.cc.html>`_.
Links
-----
* `Mapping High Level Constructs to LLVM IR <http://llvm.lyngvig.org/Articles/Mapping-High-Level-Constructs-to-LLVM-IR>`_
* `IRBuilder Sample <https://github.com/eliben/llvm-clang-samples/blob/master/src_llvm/experimental/build_llvm_ir.cpp>`_
* `Using MCJIT with Kaleidoscope <http://blog.llvm.org/2013/07/using-mcjit-with-kaleidoscope-tutorial.html>`_
* `Object format issue on Windows <http://lists.cs.uiuc.edu/pipermail/llvmdev/2013-December/068407.html>`_
* `ExecutionEngine::addGlobalMapping() bug in MCJIT <http://llvm.org/bugs/show_bug.cgi?id=20656>`_
* `LLVM Notes <http://nondot.org/sabre/LLVMNotes/>`_
* `Implementing Domain-Specific Languages with LLVM <http://cs.swan.ac.uk/~csdavec/FOSDEM12/DSLsWithLLVM.pdf>`_.

@ -1,167 +0,0 @@
LLVM Type Based Alias Analysis
==============================
When a Lua opcode involves a call to a Lua function, the Lua stack may be reallocated. So then the base pointer which points to the function's
base stack position must be refreshed.
To keep compilation simple I coded the compiler so that at the beginning of each opcode the base pointer is reloaded. My assumption was that
the LLVM optimizer will realise that the base pointer hasn't changed and so the loads are redundant and can be removed. However to my surprise I found that this
is not the case.
The main difference between the IR I was generating and that produced by Clang was that Clang generated IR appeared to be decorated
by tbaa metadata. Example::
%base2 = getelementptr inbounds %struct.CallInfoLua* %0, i32 0, i32 4, i32 0
%1 = load %struct.TValue** %base2, align 4, !tbaa !12
Here the ``!tbaa !12`` refers to a tbaa metadata entry.
I won't show the Clang generated tbaa metadata here, but here is how I added similar support in Ravi. The required steps are:
1. Create tbaa metadata mappings for the types in the system.
2. Annotate Load and Store instructions with tbaa references.
Creating TBAA Metadata
----------------------
Firstly you need an MDBuilder instance. So you need to include following headers::
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
We can create an MDBuilder instance like this::
llvm::MDBuilder mdbuilder(llvm::getGlobalContext());
The TBAA nodes hang off a root node. So we create that next::
llvm::MDNode *tbaa_root;
// Do what Clang does
tbaa_root = mdbuilder.createTBAARoot("Simple C / C++ TBAA");
Next we need to create some simple scalar types. We only need one type per size, so that means we don't need
``long long`` and ``double`` - either one will do. We create these scalar types as follows::
llvm::MDNode *tbaa_charT;
llvm::MDNode *tbaa_shortT;
llvm::MDNode *tbaa_intT;
llvm::MDNode *tbaa_longlongT;
llvm::MDNode *tbaa_pointerT;
//!4 = metadata !{metadata !"omnipotent char", metadata !5, i64 0}
tbaa_charT = mdbuilder.createTBAAScalarTypeNode("omnipotent char", tbaa_root, 0);
//!3 = metadata !{metadata !"any pointer", metadata !4, i64 0}
tbaa_pointerT = mdbuilder.createTBAAScalarTypeNode("any pointer", tbaa_charT, 0);
//!10 = metadata !{metadata !"short", metadata !4, i64 0}
tbaa_shortT = mdbuilder.createTBAAScalarTypeNode("short", tbaa_charT, 0);
//!11 = metadata !{metadata !"int", metadata !4, i64 0}
tbaa_intT = mdbuilder.createTBAAScalarTypeNode("int", tbaa_charT, 0);
//!9 = metadata !{metadata !"long long", metadata !4, i64 0}
tbaa_longlongT = mdbuilder.createTBAAScalarTypeNode("long long", tbaa_charT, 0);
The second argument to ``createTBAAScalarTypeNode()`` is the parent node. Note the hierarchy here::
+ root
|
+--+ char
|
+--+-- any pointer
|
+-- short
|
+-- int
|
+-- long long
This is how Clang has it defined.
Next we need to define aggregate (struct) types. The API we need for this is ``createTBAAStructTypeNode()``.
This method accepts a vector of ``std::pair<llvm::MDNode *, uint64_t>`` objects - each element in the vector defines
a field in the struct. The integer parameter needs to be the offset of the field within the struct. Interestingly
Clang generates offsets that indicate pointers are being treated as 32-bit quantities - even though I ran this on
a 64-bit machine. So I guess that as long as we consistently use the size then this doesn't matter. The sizes used
by Clang are:
* char - 1 byte
* short - 2 bytes
* int - 4 bytes
* pointer - 4 bytes
* long long - 8 bytes
Another interesting thing is that padding needs to be accounted for.
So now lets look at how to map following struct::
struct CallInfoL { /* only for Lua functions */
struct TValue *base; /* base for this function */
const unsigned int *savedpc;
ptrdiff_t dummy;
};
We map this as::
llvm::MDNode *tbaa_CallInfo_lT;
//!14 = metadata !{metadata !"CallInfoL", metadata !3, i64 0, metadata !3, i64 4, metadata !9, i64 8}
std::vector<std::pair<llvm::MDNode *, uint64_t> > nodes;
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_pointerT, 0));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_pointerT, 4));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_longlongT, 8));
tbaa_CallInfo_lT = mdbuilder.createTBAAStructTypeNode("CallInfo_l", nodes);
To illustrate how a structure is referenced as a field in another lets also look at::
struct CallInfo {
struct TValue *func; /* function index in the stack */
struct TValue *top; /* top for this function */
struct CallInfo *previous, *next; /* dynamic call link */
struct CallInfoL l;
ptrdiff_t extra;
short nresults; /* expected number of results from this function */
unsigned char callstatus;
};
We have a ``CallInfoL`` as the type of a field within the struct. Therefore::
llvm::MDNode *tbaa_CallInfoT;
//!13 = metadata !{metadata !"CallInfo",
// metadata !3, i64 0, metadata !3, i64 4, metadata !3, i64 8,
// metadata !3, i64 12, metadata !14, i64 16, metadata !9, i64 32,
// metadata !10, i64 40, metadata !4, i64 42}
nodes.clear();
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_pointerT, 0));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_pointerT, 4));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_pointerT, 8));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_pointerT, 12));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_CallInfo_lT, 16));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_longlongT, 32));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_shortT, 40));
nodes.push_back(std::pair<llvm::MDNode*, uint64_t>(tbaa_charT, 42));
tbaa_CallInfoT = mdbuilder.createTBAAStructTypeNode("CallInfo", nodes);
Decorating Load and Store instructions
--------------------------------------
So now we have created TBAA metadata for two struct types.
Next we need to see how we use these in Load and Store instructions. Lets assume we need to load the pointer
stored in ``Callinfo.top``. In order to decorate the Load instruction with tbaa we need to create
a Struct Tag Node - which is like a path node. Here it is::
llvm::MDNode *tbaa_CallInfo_topT;
tbaa_CallInfo_topT = mdbuilder.createTBAAStructTagNode(tbaa_CallInfoT, tbaa_pointerT, 4);
Above is saying that the field ``top`` in struct ``CallInfo`` is a pointer at offset 4.
Armed with this we can code::
llvm::Value *callinfo_top = /* GEP instruction */
llvm::Instruction *top = Builder.CreateLoad(callinfo_top);
top->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_CallInfo_topT);
Links
-----
* `TypeBasedAliasAnalysis code <http://llvm.org/docs/doxygen/html/TypeBasedAliasAnalysis_8cpp_source.html>`_.
* `IR documentation on tbaa metadata <http://llvm.org/docs/LangRef.html#tbaa-metadata>`_.
* `Embedded metadata <http://nondot.org/sabre/LLVMNotes/EmbeddedMetadata.txt>`_.

@ -1,222 +0,0 @@
LLVM First Steps
================
Note that the discussion below is for LLVM 3.5.1.
Although there appears to be a lot of documentation in the LLVM site surprisingly some basic information is hard to find.
The main source of guidance for creating a JIT is in the example toy language
`Kaleidoscope <https://github.com/llvm-mirror/llvm/blob/master/examples/Kaleidoscope/MCJIT/complete/toy.cpp>`_. But here too
there are several versions - so you have to pick the right version that is compatible with the LLVM version you
are using.
A Lua JITed function will execute in the context of Lua. So it needs to be able to access the ``lua_State`` and its various
structures. So I wanted a sample that demonstrates passing a pointer to a structure and accessing it within the JITed
function.
The initial test program I created is meant to be a "hello world" type test but covering the functionility described above.
The test I want to run is::
// Creating a function that takes pointer to struct as argument
// The function gets value from one of the fields in the struct
// And returns it
// The equivalent C program is:
//
// extern int printf(const char *, ...);
//
// struct GCObject {
// struct GCObject *next;
// unsigned char a;
// unsigned char b;
// };
//
// int testfunc(struct GCObject *obj) {
// printf("value = %d\n", obj->a);
// return obj->a;
// }
You can view the test program at `test_llvm.cpp <https://github.com/dibyendumajumdar/ravi/blob/master/tests/test_llvm.cpp>`_.
It is also reproduced below.
I used the new ``MCJIT`` engine in my test. It seems that this engine compiles modules rather than individual
functions - and once compiled a module cannot be modified. So in the Lua context we need to create a new module
everytime we JIT compile a function - or alternatively we JIT compile a whole Lua source file including all its
functions into a single module.
I found the blog post `Using MCJIT with Kaleidoscope
<http://blog.llvm.org/2013/07/using-mcjit-with-kaleidoscope-tutorial.html>`_ useful in understanding some
finer points about using ``MCJIT``.
The Lua GCObject structure in lobject.h we need is::
typedef struct RaviGCObject {
struct RaviGCObject *next;
unsigned char b1;
unsigned char b2;
} RaviGCObject;
Our prototype for the JITted function::
typedef int (*myfunc_t)(RaviGCObject *);
Get global context - not sure what the impact is of sharing::
llvm::LLVMContext &context = llvm::getGlobalContext();
Module is the translation unit::
std::unique_ptr<llvm::Module> theModule =
std::unique_ptr<llvm::Module>(new llvm::Module("ravi", context));
llvm::Module *module = theModule.get();
llvm::IRBuilder<> builder(context);
On Windows we get error saying incompatible object format
Reading posts on mailining lists I found that the issue is that COEFF
format is not supported and therefore we need to set -elf as the object
format::
#ifdef _WIN32
auto triple = llvm::sys::getProcessTriple();
module->setTargetTriple(triple + "-elf");
#endif
create a GCObject structure as defined in lobject.h::
llvm::StructType *structType =
llvm::StructType::create(context, "RaviGCObject");
llvm::PointerType *pstructType =
llvm::PointerType::get(structType, 0); // pointer to RaviGCObject
std::vector<llvm::Type *> elements;
elements.push_back(pstructType);
elements.push_back(llvm::Type::getInt8Ty(context));
elements.push_back(llvm::Type::getInt8Ty(context));
structType->setBody(elements);
structType->dump();
Create printf declaration::
std::vector<llvm::Type *> args;
args.push_back(llvm::Type::getInt8PtrTy(context));
// accepts a char*, is vararg, and returns int
llvm::FunctionType *printfType =
llvm::FunctionType::get(builder.getInt32Ty(), args, true);
llvm::Constant *printfFunc =
module->getOrInsertFunction("printf", printfType);
Create the testfunc()::
args.clear();
args.push_back(pstructType);
llvm::FunctionType *funcType =
llvm::FunctionType::get(builder.getInt32Ty(), args, false);
llvm::Function *mainFunc = llvm::Function::Create(
funcType, llvm::Function::ExternalLinkage, "testfunc", module);
llvm::BasicBlock *entry =
llvm::BasicBlock::Create(context, "entrypoint", mainFunc);
builder.SetInsertPoint(entry);
The printf format string::
llvm::Value *formatStr = builder.CreateGlobalStringPtr("value = %d\n");
Get the first argument which is RaviGCObject *::
auto argiter = mainFunc->arg_begin();
llvm::Value *arg1 = argiter++;
arg1->setName("obj");
Now we need a GEP for the second field in RaviGCObject::
std::vector<llvm::Value *> values;
llvm::APInt zero(32, 0);
llvm::APInt one(32, 1);
// This is the array offset into RaviGCObject*
values.push_back(
llvm::Constant::getIntegerValue(llvm::Type::getInt32Ty(context), zero));
// This is the field offset
values.push_back(
llvm::Constant::getIntegerValue(llvm::Type::getInt32Ty(context), one));
Create the GEP value::
llvm::Value *arg1_a = builder.CreateGEP(arg1, values, "ptr");
Now retrieve the data from the pointer address::
llvm::Value *tmp1 = builder.CreateLoad(arg1_a, "a");
As the retrieved value is a byte - convert to int i::
llvm::Value *tmp2 =
builder.CreateZExt(tmp1, llvm::Type::getInt32Ty(context), "i");
Call the printf function::
values.clear();
values.push_back(formatStr);
values.push_back(tmp2);
builder.CreateCall(printfFunc, values);
return i::
builder.CreateRet(tmp2);
module->dump();
Lets create the MCJIT engine::
std::string errStr;
auto engine = llvm::EngineBuilder(module)
.setErrorStr(&errStr)
.setEngineKind(llvm::EngineKind::JIT)
.setUseMCJIT(true)
.create();
if (!engine) {
llvm::errs() << "Failed to construct MCJIT ExecutionEngine: " << errStr
<< "\n";
return 1;
}
Now lets compile our function into machine code::
std::string funcname = "testfunc";
myfunc_t funcptr = (myfunc_t)engine->getFunctionAddress(funcname);
if (funcptr == nullptr) {
llvm::errs() << "Failed to obtain compiled function\n";
return 1;
}
Run the function and test results::
RaviGCObject obj = {NULL, 42, 65};
int ans = funcptr(&obj);
printf("The answer is %d\n", ans);
return ans == 42 ? 0 : 1;
Accessing ``extern`` functions from JIT compiled code
-----------------------------------------------------
The JITed function needs to access ``extern`` Lua functions. We need a way to map these to make these visible to the JITed code. Simply declaring
the functions ``extern`` only appears to work if the functios are available as exported symbols in dynamic libraries, e.g. the call to
``printf`` above.
From reading posts on the subject it appears that the way to do this is to add a global mapping in the ``ExecutionEngine`` by calling the
``addGlobalMapping()`` method. However this doesn't work with MCJIT due to a bug! So we need to use a workaround. Apparently there are two
solutions:
* Create a custom memory manager that resolves the ``extern`` functions.
* Add the symbol to the global symbols by calling ``llvm::sys::DynamicLibrary::AddSymbol()``.
I am using the latter approach for now.
Memory Management in LLVM
-------------------------
Curiously LLVM docs do not say much about how memory should be managed. I am still trying to figure this out, but in general it seems that there is
hierarchy of ownership. Example: ``ExecutionEngine`` owns the ``Module``. By deleting the parent the 'owned' objects are automatically
deleted.
Links
-----
* `Object format issue on Windows <http://lists.cs.uiuc.edu/pipermail/llvmdev/2013-December/068407.html>`_
* `ExecutionEngine::addGlobalMapping() bug in MCJIT <http://llvm.org/bugs/show_bug.cgi?id=20656>`_
* `LLVM Notes <http://nondot.org/sabre/LLVMNotes/>`_

@ -1,160 +0,0 @@
===================================
Building Ravi with LLVM JIT backend
===================================
.. contents:: Table of Contents
:depth: 2
:backlinks: top
Quick build without JIT
=======================
A Makefile is supplied for a simple build without the JIT on Unix platforms. Just run ``make`` and follow instructions. You may need to customize the Makefiles.
For building Ravi with JIT options please read on.
Build Dependencies
==================
* `CMake <https://cmake.org/>`_ is required
* On Windows you will need Visual Studio 2017 Community edition
* LLVM versions >= 3.5
LLVM JIT Backend
================
Following versions of LLVM work with Ravi.
* LLVM 3.7, 3.8, 3.9, 4.0, 5.0, 6.0, 8.0.1, 9.0.1, 10.0
* LLVM 7.0 was skipped because of unstable ORC api changes
* LLVM 3.5 and 3.6 should also work but have not been recently tested
Unless otherwise noted the instructions below should work for LLVM 3.9 and later.
Since LLVM 5.0 Ravi has begun to use the new ORC JIT apis. These apis are more memory efficient
compared to the MCJIT apis because they release the Module IR as early as possible, whereas with
MCJIT the Module IR hangs around as long as the compiled code is held. Because of this significant
improvement, I recommend using the latest version of LLVM.
Building LLVM on Windows
------------------------
I built LLVM from source. I used the following sequence from the VS2017 command window::
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX=c:\Software\llvm801 -DLLVM_TARGETS_TO_BUILD="X86" -G "Visual Studio 15 2017 Win64" ..
I then opened the generated solution in VS2017 and performed a INSTALL build from there. Above will build the 64-bit version of LLVM libraries. To build a 32-bit version omit the ``Win64`` parameter.
.. note:: Note that if you perform a **Release** build of LLVM then you will also need to do a **Release** build of Ravi otherwise you will get link errors. Ditto for **Debug** builds.
Building LLVM on Ubuntu or Redhat
---------------------------------
The approach is similar to that described for MAC OS X below.
Building LLVM on MAC OS X
-------------------------
I am using Max OSX Mojave. Pre-requisites are XCode and CMake.
Ensure cmake is on the path.
Assuming that LLVM source has been extracted to ``$HOME/llvm-8.0.1.src`` I follow these steps::
cd $HOME/llvm-8.0.1.src
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/Software//llvm801 -DLLVM_TARGETS_TO_BUILD="X86" ..
make install
Building Ravi with LLVM JIT backend enabled
-------------------------------------------
I am developing Ravi using Visual Studio 2017 Community Edition on Windows 10 64bit, gcc on Linux 64-bit, and clang/Xcode on MAC OS X. I was also able to successfully build a Ubuntu version on Windows 10 using the newly released Ubuntu/Linux sub-system for Windows 10.
.. note:: Location of cmake files prior to LLVM 3.9 was ``$LLVM_INSTALL_DIR/share/llvm/cmake``.
Assuming that LLVM has been installed as described above, then on Windows I invoke the cmake config as follows::
cd build
cmake -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -DLLVM_DIR=c:\Software\llvm801\lib\cmake\llvm -G "Visual Studio 15 2017 Win64" ..
I then open the solution in VS2017 and do a build from there.
On Ubuntu I use::
cd build
cmake -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/Software/ravi -DLLVM_DIR=$HOME/Software/llvm801/lib/cmake/llvm -DCMAKE_BUILD_TYPE=Release -G "Unix Makefiles" ..
make install
Note that on a clean install of Ubuntu 15.10 I had to install following packages:
* cmake
* git
* libreadline-dev
On MAC OS X I use::
cd build
cmake -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/Software/ravi -DLLVM_DIR=$HOME/Software/llvm801/lib/cmake/llvm -DCMAKE_BUILD_TYPE=Release ..
make install
Building without JIT
====================
You can omit ``-DLLVM_JIT=ON`` options to build Ravi with a null JIT implementation.
Building Static Libraries
=========================
By default the build generates a shared library for Ravi. You can choose to create a static library and statically linked executables by supplying the argument ``-DSTATIC_BUILD=ON`` to CMake.
JIT API
-------
auto mode
in this mode the compiler decides when to compile a Lua function. The current implementation is very simple -
any Lua function call is checked to see if the bytecodes contained in it can be compiled. If this is true then
the function is compiled provided either a) function has a fornum loop, or b) it is largish (greater than 150 bytecodes)
or c) it is being executed many times (> 50). Because of the simplistic behaviour performance the benefit of JIT
compilation is only available if the JIT compiled functions will be executed many times so that the cost of JIT
compilation can be amortized.
manual mode
in this mode user must explicitly request compilation. This is the default mode. This mode is suitable for library
developers who can pre compile the functions in library module table.
A JIT api is available with following functions:
``ravi.jit([b])``
returns enabled setting of JIT compiler; also enables/disables the JIT compiler; defaults to true
``ravi.auto([b [, min_size [, min_executions]]])``
returns setting of auto compilation and compilation thresholds; also sets the new settings if values are supplied; defaults are false, 150, 50.
``ravi.compile(func_or_table[, options])``
compiles a Lua function (or functions if a table is supplied) if possible, returns ``true`` if compilation was
successful for at least one function. ``options`` is an optional table with compilation options - in particular
``omitArrayGetRangeCheck`` - which disables range checks in array get operations to improve performance in some cases.
Note that at present if the first argument is a table of functions and has more than 100 functions then only the
first 100 will be compiled. You can invoke compile() repeatedly on the table until it returns false. Each
invocation leads to a new module being created; any functions already compiled are skipped.
``ravi.iscompiled(func)``
returns the JIT status of a function
``ravi.dumplua(func)``
dumps the Lua bytecode of the function
``ravi.dumpir(func)``
dumps the IR of the compiled function (only if function was compiled; only available in LLVM 4.0 and earlier)
``ravi.dumpasm(func)``
(deprecated) dumps the machine code using the currently set optimization level (only if function was compiled; only available in LLVM version 4.0 and earlier)
``ravi.optlevel([n])``
sets LLVM optimization level (0, 1, 2, 3); defaults to 2. These levels are handled by reusing LLVMs default pass definitions which are geared towards C/C++ programs, but appear to work well here. If level is set to 0, then an attempt is made to use fast instruction selection to further speed up compilation.
``ravi.sizelevel([n])``
sets LLVM size level (0, 1, 2); defaults to 0
``ravi.tracehook([b])``
Enables support for line hooks via the debug api. Note that enabling this option will result in inefficient JIT as a call to a C function will be inserted at beginning of every Lua bytecode boundary; use this option only when you want to use the debug api to step through code line by line
``ravi.verbosity([b])``
Controls the amount of verbose messages generated during compilation.
Performance
===========
For performance benchmarks please visit the `Ravi Performance Benchmarks <http://the-ravi-programming-language.readthedocs.org/en/latest/ravi-benchmarks.html>`_ page.
To obtain the best possible performance, types must be annotated so that Ravi's JIT compiler can generate efficient code.
Additionally function calls are expensive - as the JIT compiler cannot inline function calls, all function calls go via the Lua call protocol which has a large overhead. This is true for both Lua functions and C functions. For best performance avoid function calls inside loops.
Testing
=======
I test the build by running a modified version of Lua 5.3.3 test suite. These tests are located in the ``lua-tests`` folder. Additionally I have ravi specific tests in the ``ravi-tests`` folder. There is a also a travis build that occurs upon commits - this build runs the tests as well.
.. note:: To thoroughly test changes, you need to invoke CMake with ``-DCMAKE_BUILD_TYPE=Debug`` option. This turns on assertions, memory checking, and also enables an internal module used by Lua tests.

@ -1,707 +0,0 @@
Lua Types in LLVM
=================
We need to map Lua types to equivalent type definitions in LLVM. In Ravi we do hold all the type definitions in a struct as shown below::
struct LuaLLVMTypes {
llvm::Type *C_intptr_t;
llvm::Type *C_size_t;
llvm::Type *C_ptrdiff_t;
llvm::Type *lua_NumberT;
llvm::Type *lua_IntegerT;
llvm::Type *lua_UnsignedT;
llvm::Type *lua_KContextT;
llvm::FunctionType *lua_CFunctionT;
llvm::PointerType *plua_CFunctionT;
llvm::FunctionType *lua_KFunctionT;
llvm::PointerType *plua_KFunctionT;
llvm::FunctionType *lua_HookT;
llvm::PointerType *plua_HookT;
llvm::FunctionType *lua_AllocT;
llvm::PointerType *plua_AllocT;
llvm::Type *l_memT;
llvm::Type *lu_memT;
llvm::Type *lu_byteT;
llvm::Type *L_UmaxalignT;
llvm::Type *C_pcharT;
llvm::Type *C_intT;
llvm::StructType *lua_StateT;
llvm::PointerType *plua_StateT;
llvm::StructType *global_StateT;
llvm::PointerType *pglobal_StateT;
llvm::StructType *ravi_StateT;
llvm::PointerType *pravi_StateT;
llvm::StructType *GCObjectT;
llvm::PointerType *pGCObjectT;
llvm::StructType *ValueT;
llvm::StructType *TValueT;
llvm::PointerType *pTValueT;
llvm::StructType *TStringT;
llvm::PointerType *pTStringT;
llvm::PointerType *ppTStringT;
llvm::StructType *UdataT;
llvm::StructType *TableT;
llvm::PointerType *pTableT;
llvm::StructType *UpvaldescT;
llvm::PointerType *pUpvaldescT;
llvm::Type *ravitype_tT;
llvm::StructType *LocVarT;
llvm::PointerType *pLocVarT;
llvm::Type *InstructionT;
llvm::PointerType *pInstructionT;
llvm::StructType *LClosureT;
llvm::PointerType *pLClosureT;
llvm::PointerType *ppLClosureT;
llvm::PointerType *pppLClosureT;
llvm::StructType *RaviJITProtoT;
llvm::PointerType *pRaviJITProtoT;
llvm::StructType *ProtoT;
llvm::PointerType *pProtoT;
llvm::PointerType *ppProtoT;
llvm::StructType *UpValT;
llvm::PointerType *pUpValT;
llvm::StructType *CClosureT;
llvm::PointerType *pCClosureT;
llvm::StructType *TKeyT;
llvm::PointerType *pTKeyT;
llvm::StructType *NodeT;
llvm::PointerType *pNodeT;
llvm::StructType *lua_DebugT;
llvm::PointerType *plua_DebugT;
llvm::StructType *lua_longjumpT;
llvm::PointerType *plua_longjumpT;
llvm::StructType *MbufferT;
llvm::StructType *stringtableT;
llvm::PointerType *StkIdT;
llvm::StructType *CallInfoT;
llvm::StructType *CallInfo_cT;
llvm::StructType *CallInfo_lT;
llvm::PointerType *pCallInfoT;
llvm::FunctionType *jitFunctionT;
llvm::FunctionType *luaD_poscallT;
};
The actual definition of the types above is shown below::
static_assert(std::is_floating_point<lua_Number>::value &&
sizeof(lua_Number) == sizeof(double),
"lua_Number is not a double");
lua_NumberT = llvm::Type::getDoubleTy(context);
static_assert(std::is_integral<lua_Integer>::value,
"lua_Integer is not an integer type");
lua_IntegerT = llvm::Type::getIntNTy(context, sizeof(lua_Integer) * 8);
static_assert(sizeof(lua_Integer) == sizeof(lua_Unsigned),
"lua_Integer and lua_Unsigned are of different size");
lua_UnsignedT = lua_IntegerT;
C_intptr_t = llvm::Type::getIntNTy(context, sizeof(intptr_t) * 8);
C_size_t = llvm::Type::getIntNTy(context, sizeof(size_t) * 8);
C_ptrdiff_t = llvm::Type::getIntNTy(context, sizeof(ptrdiff_t) * 8);
C_intT = llvm::Type::getIntNTy(context, sizeof(int) * 8);
static_assert(sizeof(size_t) == sizeof(lu_mem),
"lu_mem size is not same as size_t");
lu_memT = C_size_t;
static_assert(sizeof(ptrdiff_t) == sizeof(l_mem),
"l_mem size is not same as ptrdiff_t");
l_memT = C_ptrdiff_t;
static_assert(sizeof(L_Umaxalign) == sizeof(double),
"L_Umaxalign is not same size as double");
L_UmaxalignT = llvm::Type::getDoubleTy(context);
lu_byteT = llvm::Type::getInt8Ty(context);
C_pcharT = llvm::Type::getInt8PtrTy(context);
InstructionT = C_intT;
pInstructionT = llvm::PointerType::get(InstructionT, 0);
lua_StateT = llvm::StructType::create(context, "ravi.lua_State");
plua_StateT = llvm::PointerType::get(lua_StateT, 0);
lua_KContextT = C_ptrdiff_t;
std::vector<llvm::Type *> elements;
elements.push_back(plua_StateT);
lua_CFunctionT = llvm::FunctionType::get(C_intT, elements, false);
plua_CFunctionT = llvm::PointerType::get(lua_CFunctionT, 0);
jitFunctionT = lua_CFunctionT;
elements.clear();
elements.push_back(plua_StateT);
elements.push_back(C_intT);
elements.push_back(lua_KContextT);
lua_KFunctionT = llvm::FunctionType::get(C_intT, elements, false);
plua_KFunctionT = llvm::PointerType::get(lua_KFunctionT, 0);
elements.clear();
elements.push_back(llvm::Type::getInt8PtrTy(context));
elements.push_back(llvm::Type::getInt8PtrTy(context));
elements.push_back(C_size_t);
elements.push_back(C_size_t);
lua_AllocT = llvm::FunctionType::get(llvm::Type::getInt8PtrTy(context),
elements, false);
plua_AllocT = llvm::PointerType::get(lua_AllocT, 0);
lua_DebugT = llvm::StructType::create(context, "ravi.lua_Debug");
plua_DebugT = llvm::PointerType::get(lua_DebugT, 0);
elements.clear();
elements.push_back(plua_StateT);
elements.push_back(plua_DebugT);
lua_HookT = llvm::FunctionType::get(llvm::Type::getInt8PtrTy(context),
elements, false);