compile against LLVM 3.7

CMakeLists.txt

@@ -99,10 +99,40 @@ else()
   set(EXTRA_LIBRARIES m dl readline)
 endif()
 
-llvm_map_components_to_libnames(llvm_libs all)
+#llvm_map_components_to_libnames(llvm_libs all)
 #llvm_map_components_to_libnames(llvm_libs Support Core CodeGen AsmPrinter BitReader TransformUtils  
 # InstCombine ScalarOpts Analysis ipa Object ExecutionEngine MC MCParser MCJIT ProfileData ipo
 # RuntimeDyld Target X86CodeGen X86AsmParser X86Disassembler X86AsmPrinter X86Desc X86Info X86Utils AsmParser)
+
+llvm_map_components_to_libnames(llvm_libs 
+  Analysis 
+  Core 
+  CodeGen 
+  AsmParser 
+  AsmPrinter 
+  BitReader 
+  ExecutionEngine 
+  InstCombine 
+  ipa 
+  ipo  
+  MC 
+  MCJIT 
+  MCParser 
+  Object  
+  RuntimeDyld 
+  ScalarOpts 
+  Support 
+  Target 
+  TransformUtils 
+  X86CodeGen 
+  X86AsmParser 
+  X86Disassembler 
+  X86AsmPrinter 
+  X86Desc 
+  X86Info 
+  X86Utils 
+  )
+
 message(STATUS "llvm_libs ${llvm_libs}")
 
 #Main library

include/ravi_llvmcodegen.h

@@ -530,7 +530,8 @@ public:
   // The p->ravi_jit structure will be updated
   // Note that if a function fails to compile then
   // a flag is set so that it doesn't get compiled again
-  void compile(lua_State *L, Proto *p, bool doDump = false, bool doVerify=true);
+  void compile(lua_State *L, Proto *p, bool doDump = false,
+               bool doVerify = true);
 
   // We can only compile a subset of op codes
   // and not all features are supported

include/ravijit.h

@@ -37,7 +37,8 @@ int raviV_initjit(struct lua_State *L);
 void raviV_close(struct lua_State *L);
 
 /* Compile the given function if possible */
-int raviV_compile(struct lua_State *L, struct Proto *p, int manual_request, int dump);
+int raviV_compile(struct lua_State *L, struct Proto *p, int manual_request,
+                  int dump);
 
 /* Free the JIT structures associated with the prototype */
 void raviV_freeproto(struct lua_State *L, struct Proto *p);

include/ravillvm.h

@@ -44,7 +44,11 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IR/Metadata.h"
+#if LLVM_VERSION_MINOR < 7
 #include "llvm/PassManager.h"
+#else
+#include "llvm/IR/LegacyPassManager.h"
+#endif
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Support/TargetSelect.h"
@@ -59,7 +63,7 @@
 #include <vector>
 #include <memory>
 
-#if defined(_WIN32) 
+#if defined(_WIN32)
 #if defined(LUA_BUILD_AS_DLL)
 #define RAVI_API __declspec(dllexport)
 #else
@@ -90,8 +94,9 @@ public:
   // Add declaration for an extern function that is not
   // loaded dynamically - i.e., is part of the the executable
   // and therefore not visible at runtime by name
-  virtual llvm::Constant *addExternFunction(llvm::FunctionType *type, void *address,
-                                    const std::string &name) = 0;
+  virtual llvm::Constant *addExternFunction(llvm::FunctionType *type,
+                                            void *address,
+                                            const std::string &name) = 0;
 
   virtual const std::string &name() const = 0;
   virtual llvm::Function *function() const = 0;
@@ -103,9 +108,10 @@ public:
 
 protected:
   RaviJITFunction() {}
+
 private:
-  RaviJITFunction(const RaviJITFunction&) = delete;
-  RaviJITFunction& operator=(const RaviJITFunction&) = delete;
+  RaviJITFunction(const RaviJITFunction &) = delete;
+  RaviJITFunction &operator=(const RaviJITFunction &) = delete;
 };
 
 // Ravi's JIT State
@@ -115,9 +121,10 @@ public:
   virtual ~RaviJITState() {}
 
   // Create a function of specified type and linkage
-  virtual RaviJITFunction *createFunction(llvm::FunctionType *type,
-                                  llvm::GlobalValue::LinkageTypes linkage,
-                                  const std::string &name) = 0;
+  virtual RaviJITFunction *
+  createFunction(llvm::FunctionType *type,
+                 llvm::GlobalValue::LinkageTypes linkage,
+                 const std::string &name) = 0;
 
   virtual void dump() = 0;
   virtual llvm::LLVMContext &context() = 0;
@@ -133,19 +140,19 @@ public:
   virtual void set_mincodesize(int) = 0;
   virtual int get_minexeccount() const = 0;
   virtual void set_minexeccount(int) = 0;
-    
+
 protected:
   RaviJITState() {}
+
 private:
-  RaviJITState(const RaviJITState&) = delete;
-  RaviJITState& operator=(const RaviJITState&) = delete;
+  RaviJITState(const RaviJITState &) = delete;
+  RaviJITState &operator=(const RaviJITState &) = delete;
 };
 
 class RAVI_API RaviJITStateFactory {
 public:
   static std::unique_ptr<RaviJITState> newJITState();
 };
-
 }
 
 #endif

readthedocs/llvm-notes.rst

@@ -73,8 +73,11 @@ Then as I intend to use the ``u.l`` field more often, I used the following defin
 
 JIT Compilation Error on Windows
 --------------------------------
+Note: The latest LLVM version from LLVM source repository appears to support COEFF format. So below
+applies to version 3.6.x and 3.5.x.
+
 On Windows when we attempt to JIT compile we get an error saying incompatible object format
-Reading posts on mailining lists I found that the issue is that COEFF
+Reading posts on mailing lists I found that the issue is that COEFF
 format is not supported and therefore we need to set -elf as the object
 format::
 
@@ -153,42 +156,6 @@ The LLVM documentation does not provide guidance on how the optimization passes
 
 Fortunately it seems that there is a `PassManagerBuilder <http://llvm.org/docs/doxygen/html/classllvm_1_1PassManagerBuilder.html>`_ component that allows easy setup of the standard passes for a C like language. Unfortunately there isn't much guidance on how to use this either. The best source of information I found was an example toy compiler by `David Chisnall <http://cs.swan.ac.uk/~csdavec/FOSDEM12/compiler.cc.html>`_. 
 
-In Ravi each function is kept in its own llvm Module. The way I setup the optimization passes is shown below::
-
-  // Create a function pass manager for this engine
-  llvm::FunctionPassManager *FPM = new llvm::FunctionPassManager(module_);
-
-  // Set up the optimizer pipeline.  Start with registering info about how the
-  // target lays out data structures.
-  #if LLVM_VERSION_MINOR > 5
-  
-  // LLVM 3.6.0 change
-  module_->setDataLayout(engine_->getDataLayout());
-  FPM->add(new llvm::DataLayoutPass());
-  
-  #else
-  
-  auto target_layout = engine_->getTargetMachine()->getDataLayout();
-  module_->setDataLayout(target_layout);
-  FPM->add(new llvm::DataLayoutPass(*engine_->getDataLayout()));
-  
-  #endif
-  
-  llvm::PassManagerBuilder pmb;
-  pmb.OptLevel = 3;
-  pmb.SizeLevel = 0;
-  
-  pmb.populateFunctionPassManager(*FPM);
-  FPM->doInitialization();
-  FPM->run(*function_);
-  delete FPM;
-
-  llvm::PassManager *MPM = new llvm::PassManager();
-  pmb.populateModulePassManager(*MPM);
-  MPM->run(*module_);
-  delete MPM;
-
-
 Links
 -----
 * `Mapping High Level Constructs to LLVM IR <http://llvm.lyngvig.org/Articles/Mapping-High-Level-Constructs-to-LLVM-IR>`_

src/ravijit.cpp

@@ -30,7 +30,7 @@
 
 namespace ravi {
 
-// This is just to avoid initailizing LLVM repeatedly -
+// This is just to avoid initializing LLVM repeatedly -
 // see below
 static std::atomic_int init;
 
@@ -51,7 +51,7 @@ RaviJITStateImpl::RaviJITStateImpl()
     init++;
   }
   triple_ = llvm::sys::getProcessTriple();
-#ifdef _WIN32
+#if defined(_WIN32) && LLVM_VERSION_MINOR < 7
   // On Windows we get compilation error saying incompatible object format
   // Reading posts on mailing lists I found that the issue is that COEFF
   // format is not supported and therefore we need to set -elf as the object
@@ -108,9 +108,9 @@ RaviJITFunctionImpl::RaviJITFunctionImpl(
   // module per function
   std::string moduleName = "ravi_module_" + name;
   module_ = new llvm::Module(moduleName, owner->context());
-#if defined(_WIN32)
+#if defined(_WIN32) && LLVM_VERSION_MINOR < 7
   // On Windows we get error saying incompatible object format
-  // Reading posts on mailining lists I found that the issue is that COEFF
+  // Reading posts on mailing lists I found that the issue is that COEFF
   // format is not supported and therefore we need to set
   // -elf as the object format
   module_->setTargetTriple(owner->triple());
@@ -190,9 +190,21 @@ static void addMemorySanitizerPass(const llvm::PassManagerBuilder &Builder,
 #endif
 
 void RaviJITFunctionImpl::runpasses(bool dumpAsm) {
+
+#if LLVM_VERSION_MINOR == 7
+  using llvm::legacy::FunctionPassManager;
+  using llvm::legacy::PassManager;
+#else
+  using llvm::FunctionPassManager;
+  using llvm::PassManager;
+#endif
+
   // We use the PassManagerBuilder to setup optimization
   // passes - the PassManagerBuilder allows easy configuration of
   // typical C/C++ passes corresponding to O0, O1, O2, and O3 compiler options
+  // If dumpAsm is true then the generated assembly code will be
+  // dumped to stderr
+
   llvm::PassManagerBuilder pmb;
   pmb.OptLevel = owner_->get_optlevel();
   pmb.SizeLevel = owner_->get_sizelevel();
@@ -213,39 +225,51 @@ void RaviJITFunctionImpl::runpasses(bool dumpAsm) {
 #endif
   {
     // Create a function pass manager for this engine
-    std::unique_ptr<llvm::FunctionPassManager> FPM(
-        new llvm::FunctionPassManager(module_));
+    std::unique_ptr<FunctionPassManager> FPM(new FunctionPassManager(module_));
 
 // Set up the optimizer pipeline.  Start with registering info about how the
 // target lays out data structures.
-#if LLVM_VERSION_MINOR > 5
+#if LLVM_VERSION_MINOR == 6
     // LLVM 3.6.0 change
     module_->setDataLayout(engine_->getDataLayout());
     FPM->add(new llvm::DataLayoutPass());
-#else
+#elif LLVM_VERSION_MINOR == 5
     // LLVM 3.5.0
     auto target_layout = engine_->getTargetMachine()->getDataLayout();
     module_->setDataLayout(target_layout);
     FPM->add(new llvm::DataLayoutPass(*engine_->getDataLayout()));
+#elif LLVM_VERSION_MINOR == 7
+#else
+#error Unsupported LLVM version
 #endif
     pmb.populateFunctionPassManager(*FPM);
     FPM->doInitialization();
     FPM->run(*function_);
   }
 
+  std::string codestr;
   {
-    std::unique_ptr<llvm::PassManager> MPM(new llvm::PassManager());
-#if LLVM_VERSION_MINOR > 5
-    MPM->add(new llvm::DataLayoutPass());
+    // In LLVM 3.7 for some reason the string is not saved
+    // until the stream is destroyed - even though there is a
+    // flush; so we introduce a scope here to ensure destruction
+    // of the stream
+    llvm::raw_string_ostream ostream(codestr);
+#if LLVM_VERSION_MINOR < 7
+    llvm::formatted_raw_ostream formatted_stream(ostream);
 #else
+    llvm::buffer_ostream formatted_stream(ostream);
+#endif
+
+    // Also in 3.7 the pass manager seems to hold on to the stream
+    // so we need to ensure that the stream outlives the pass manager
+    std::unique_ptr<PassManager> MPM(new PassManager());
+#if LLVM_VERSION_MINOR == 6
+    MPM->add(new llvm::DataLayoutPass());
+#elif LLVM_VERSION_MINOR == 5
     MPM->add(new llvm::DataLayoutPass(*engine_->getDataLayout()));
 #endif
     pmb.populateModulePassManager(*MPM);
 
-    std::string codestr;
-    llvm::raw_string_ostream ostream(codestr);
-    llvm::formatted_raw_ostream formatted_stream(ostream);
-
     for (int i = 0; dumpAsm && i < 1; i++) {
       llvm::TargetMachine *TM = engine_->getTargetMachine();
       if (!TM) {
@@ -258,13 +282,13 @@ void RaviJITFunctionImpl::runpasses(bool dumpAsm) {
         break;
       }
     }
-
     MPM->run(*module_);
 
+    // Note that in 3.7 this flus appears to have no effect
     formatted_stream.flush();
-    if (dumpAsm && codestr.length() > 0)
-      llvm::errs() << codestr << "\n";
   }
+  if (dumpAsm && codestr.length() > 0)
+    llvm::errs() << codestr << "\n";
 }
 
 void *RaviJITFunctionImpl::compile(bool doDump) {
@@ -364,8 +388,8 @@ void raviV_close(struct lua_State *L) {
 
 // Compile a Lua function
 // If JIT is turned off then compilation is skipped
-// Compilation occurs if either auto compilation is ON  or
-// a manual compilation request was made
+// Compilation occurs if either auto compilation is ON (subject to some thresholds) 
+// or if a manual compilation request was made
 // Returns true if compilation was successful
 int raviV_compile(struct lua_State *L, struct Proto *p, int manual_request,
                   int dump) {
@@ -377,26 +401,19 @@ int raviV_compile(struct lua_State *L, struct Proto *p, int manual_request,
   if (!G->ravi_state->jit->is_enabled()) {
     return 0;
   }
-#if 0
-  // For some reason on Windows we sometimes get
-  // an exception in luaD_throw() when JIT compilation
-  // is ON.
-  bool doCompile = (G->ravi_state->jit->is_auto() || (bool)manual_request);
-#else
   bool doCompile = (bool)manual_request;
   if (!doCompile && G->ravi_state->jit->is_auto()) {
-    if (p->ravi_jit.jit_flags != 0) /* loop */
+    if (p->ravi_jit.jit_flags != 0) /* function has fornum loop, so compile */
       doCompile = true;
-    else if (p->sizecode > G->ravi_state->jit->get_mincodesize())
+    else if (p->sizecode > G->ravi_state->jit->get_mincodesize()) /* function is long so compile */
       doCompile = true;
     else {
-      if (p->ravi_jit.execution_count < G->ravi_state->jit->get_minexeccount())
+      if (p->ravi_jit.execution_count < G->ravi_state->jit->get_minexeccount()) /* function has been executed many times so compile */
         p->ravi_jit.execution_count++;
       else
         doCompile = true;
     }
   }
-#endif
   if (doCompile)
     G->ravi_state->code_generator->compile(L, p, dump != 0);
   return p->ravi_jit.jit_status == 2;
@@ -450,6 +467,8 @@ static int ravi_is_compiled(lua_State *L) {
 }
 
 // Try to JIT compile the given function
+// Optional boolean (second) parameter specifies whether
+// to dump the code generation 
 static int ravi_compile(lua_State *L) {
   int n = lua_gettop(L);
   luaL_argcheck(L, n >= 1, 1, "1 or 2 arguments expected");
@@ -457,7 +476,11 @@ static int ravi_compile(lua_State *L) {
                 "argument must be a Lua function");
   void *p = (void *)lua_topointer(L, 1);
   LClosure *l = (LClosure *)p;
-  int result = raviV_compile(L, l->p, 1, (n == 2) ? lua_toboolean(L, 2) : 0);
+  int manualRequest = 1; 
+  // Is there a second boolean parameter requesting
+  // dump of code generation?
+  int dumpAsm = (n == 2) ? lua_toboolean(L, 2) : 0;
+  int result = raviV_compile(L, l->p, manualRequest, dumpAsm);
   lua_pushboolean(L, result);
   return 1;
 }