issue #98 save current state in parser branch

.devcontainer/Dockerfile

@@ -1,7 +0,0 @@
-# [Choice] Debian / Ubuntu version: debian-10, debian-9, ubuntu-20.04, ubuntu-18.04
-ARG VARIANT=buster
-FROM mcr.microsoft.com/vscode/devcontainers/cpp:dev-${VARIANT}
-
-# [Optional] Uncomment this section to install additional packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/base.Dockerfile

@@ -1,12 +0,0 @@
-# [Choice] Debian / Ubuntu version: debian-10, debian-9, ubuntu-20.04, ubuntu-18.04
-ARG VARIANT=buster
-FROM mcr.microsoft.com/vscode/devcontainers/base:${VARIANT}
-
-# Install needed packages. Use a separate RUN statement to add your own dependencies.
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install build-essential cmake cppcheck valgrind clang lldb llvm gdb \
-    && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
-
-# [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/devcontainer.json

@@ -1,28 +0,0 @@
-{
-	"name": "C++",
-	"build": {
-		"dockerfile": "Dockerfile",
-		// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-10, debian-9, ubuntu-20.04, ubuntu-18.04
-		"args": { "VARIANT": "debian-10" }
-	},
-	"runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
-
-	// Set *default* container specific settings.json values on container create.
-	"settings": { 
-		"terminal.integrated.shell.linux": "/bin/bash"
-	},
-
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"ms-vscode.cpptools"
-	],
-
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [],
-
-	// Use 'postCreateCommand' to run commands after the container is created.
-	// "postCreateCommand": "gcc -v",
-
-	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-	"remoteUser": "vscode"
-}

.github/FUNDING.yml

@@ -0,0 +1,8 @@
+# These are supported funding model platforms
+
+github: dibyendumajumdar # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+custom: # Replace with a single custom sponsorship URL

.github/workflows/build.yml

@@ -1,48 +0,0 @@
-name: build
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-env:
-  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-  BUILD_TYPE: Release
-  
-jobs:
-  build:
-    
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Install Dependencies
-      run: sudo apt-get install -y libreadline-dev 
-
-    - name: Create Build Environment
-      run: cmake -E make_directory ${{runner.workspace}}/build
-
-    - name: Configure CMake
-      # Use a bash shell so we can use the same syntax for environment variable
-      # access regardless of the host operating system
-      shell: bash
-      working-directory: ${{runner.workspace}}/build
-      # Note the current convention is to use the -S and -B options here to specify source 
-      # and build directories, but this is only available with CMake 3.13 and higher.  
-      # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
-
-    - name: Build
-      working-directory: ${{runner.workspace}}/build
-      shell: bash
-      # Execute the build.  You can specify a specific target with "--target <NAME>"
-      run: cmake --build . --config $BUILD_TYPE
-
-    - name: Test
-      shell: bash
-      run: cd tests && sh run_tests.sh ${{runner.workspace}}/build/ravi

.travis.yml

@@ -3,18 +3,24 @@ os:
  - linux
 arch:
  - amd64
- - arm64
 compiler:
  - gcc
 cache: ccache
-dist: focal
+dist: bionic
 addons:
   apt:
     packages:
+      - g++
       - gcc
       - ccache
+install:
+ - curl https://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz | tar -xJf -
 script:
+ - mkdir $TRAVIS_BUILD_DIR/build
+ - cd $TRAVIS_BUILD_DIR/build && cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_DIR=$TRAVIS_BUILD_DIR/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04/lib/cmake/llvm -G "Unix Makefiles" -DLLVM_JIT=ON ..
+ - cd $TRAVIS_BUILD_DIR/build && make
+ - cd $TRAVIS_BUILD_DIR/lua-tests && sh ./run_travis_tests.sh $TRAVIS_BUILD_DIR/build/ravi
  - mkdir $TRAVIS_BUILD_DIR/buildmir
- - cd $TRAVIS_BUILD_DIR/buildmir && cmake -DCMAKE_BUILD_TYPE=Debug -DLTESTS=ON -DMIR_JIT=ON -G "Unix Makefiles" ..
+ - cd $TRAVIS_BUILD_DIR/buildmir && cmake -DCMAKE_BUILD_TYPE=Release -G "Unix Makefiles" -DMIR_JIT=ON ..
  - cd $TRAVIS_BUILD_DIR/buildmir && make
- - cd $TRAVIS_BUILD_DIR/tests && sh ./run_tests.sh $TRAVIS_BUILD_DIR/buildmir/ravi
+ - cd $TRAVIS_BUILD_DIR/lua-tests && sh ./run_travis_tests.sh $TRAVIS_BUILD_DIR/buildmir/ravi

CMakeLists.txt

@@ -1,22 +1,39 @@
 cmake_minimum_required(VERSION 3.12)
-project(Ravi VERSION 1.0.4 LANGUAGES C)
+project(Ravi VERSION 1.0.4 LANGUAGES C CXX)
 
-# By default MIR JIT backend is automatically enabled. To disable user must specify
-# NO_JIT=ON
+# Get access to CMake helpers
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
 
+option(LLVM_JIT "Controls whether LLVM JIT compilation will be enabled, default is OFF" OFF)
+option(OMR_JIT "Controls whether NanoJIT compilation will be enabled, default is OFF" OFF)
 option(NO_JIT "Controls whether JIT should be disabled, default is OFF" OFF)
 option(STATIC_BUILD "Build static version of Ravi, default is OFF" OFF)
 option(COMPUTED_GOTO "Controls whether the interpreter switch will use computed gotos on gcc/clang, default is ON" ON)
 option(LTESTS "Controls whether ltests are enabled in Debug mode; note requires Debug build" ON)
-option(ASAN "Controls whether address sanitizer should be enabled" OFF)
-option(RAVICOMP "Controls whether to link in RaviComp" ON)
 
-# By we enable MIR JIT
-if (NOT NO_JIT)
+if (LLVM_JIT AND OMR_JIT)
+    message(FATAL_ERROR
+            "LLVM_JIT and OMR_JIT cannot be set to ON at the same time")
+endif ()
+
+# By default on X86_64 non-Windows platforms we enable MIR JIT
+if (NOT WIN32
+        AND NOT LLVM_JIT
+        AND NOT OMR_JIT
+        AND NOT NO_JIT
+        AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
     set(MIR_JIT ON)
 endif ()
 
 if (MIR_JIT)
+    if (MSVC OR WIN32)
+        message(FATAL_ERROR "MIR_JIT is not supported when using MSVC and/or WIN32")
+    endif ()
+    if (NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)"))
+        message(FATAL_ERROR "MIR_JIT is only supported on X86-64 platforms")
+    endif ()
+    set(LLVM_JIT OFF)
+    set(OMR_JIT OFF)
     set(STATIC_BUILD OFF) # Because we need to expose the symbols in the library
 endif ()
 
@@ -33,21 +50,54 @@ if (COMPUTED_GOTO AND MSVC)
 endif ()
 
 include_directories("${PROJECT_SOURCE_DIR}/include")
+if ($ENV{CLION_IDE})
+    # CLion seems unable to handle include paths set on sources
+    include_directories("${PROJECT_SOURCE_DIR}/dmr_c/src")
+endif()
 
 # define the Lua core source files
+set(RAVI_AST_SOURCES src/ravi_ast_parse.c src/ravi_ast_print.c src/ravi_ast_typecheck.c src/ravi_ast_linearize.c
+	src/ravi_fnv_hash.c src/ravi_hash_table.c src/ravi_set.c src/ravi_int_set.c)
 set(LUA_CORE_SRCS src/lapi.c src/lcode.c src/lctype.c src/ldebug.c src/ldo.c src/ldump.c
         src/lfunc.c src/lgc.c src/llex.c src/lmem.c src/lobject.c src/lopcodes.c
         src/lparser.c src/lstate.c src/lstring.c src/ltable.c src/ltm.c src/lundump.c
         src/lvm.c src/lzio.c src/ravi_jit.c src/ltests.c src/ravi_profile.c
-        src/ravi_membuf.c src/ravi_jitshared.c src/bit.c src/ravi_alloc.c)
+        src/ravi_membuf.c src/ravi_jitshared.c src/bit.c src/ravi_alloc.c
+        ${RAVI_AST_SOURCES})
 # define the Lua library source files
 set(LUA_LIB_SRCS src/lauxlib.c src/lbaselib.c src/lbitlib.c src/lcorolib.c src/ldblib.c src/liolib.c
         src/lmathlib.c src/loslib.c src/ltablib.c src/lstrlib.c src/loadlib.c src/linit.c src/lutf8lib.c)
 set(LUA_HEADERS include/lua.h include/luaconf.h include/lualib.h include/lauxlib.h)
+# LLVM code gen
+set(LLVM_JIT_SRCS src/ravi_llvmjit.cpp src/ravi_llvmtypes.cpp
+        src/ravi_llvmcodegen.cpp src/ravi_llvmforprep.cpp src/ravi_llvmcomp.cpp
+        src/ravi_llvmreturn.cpp src/ravi_llvmload.cpp src/ravi_llvmforloop.cpp
+        src/ravi_llvmarith1.cpp src/ravi_llvmcall.cpp src/ravi_llvmtable.cpp
+        src/ravi_llvmarith2.cpp src/ravi_llvmtforcall.cpp src/ravi_llvmrest.cpp)
+# MIR sources
+set(MIR_HEADERS mir/mir.h mir/mir-gen.h mir/mir-varr.h mir/mir-dlist.h mir/mir-htab.h
+        mir/mir-hash.h mir/mir-bitmap.h)
+set(MIR_SRCS mir/mir.c mir/mir-gen.c)
+set(C2MIR_SRCS mir/c2mir/c2mir.c)
+# OMR code gen
+set(OMR_JIT_SRCS src/ravi_omrjit.c src/ravi_omrjitapi.c)
+# MIR code gen
 set(MIR_JIT_SRCS src/ravi_mirjit.c)
 set(NO_JIT_SRCS src/ravi_nojit.c)
+# Common stuff we need even if we don't use dmrC.
+set(DMR_C_HEADERS_COMMON dmr_c/src/allocate.h dmr_c/src/port.h dmr_c/src/ptrlist.h)
+set(DMR_C_SRCS_COMMON dmr_c/src/allocate.c dmr_c/src/ptrlist.c)
+# dmrC Optional sources
+set(RAVI_C_SYMBOL_PARSER src/ravi_dmrc_parsesymbols.c)
+set(DMR_C_SRCS_OPT dmr_c/src/builtin.c dmr_c/src/char.c dmr_c/src/expression.c dmr_c/src/evaluate.c
+        dmr_c/src/expand.c dmr_c/src/inline.c dmr_c/src/lib.c dmr_c/src/linearize.c
+        dmr_c/src/liveness.c dmr_c/src/parse.c dmr_c/src/target.c dmr_c/src/tokenize.c
+        dmr_c/src/pre-process.c dmr_c/src/scope.c dmr_c/src/show-parse.c dmr_c/src/symbol.c
+        dmr_c/src/walksymbol.c)
+set(DMR_C_HEADERS_OPT dmr_c/src/char.h dmr_c/src/expression.h dmr_c/src/flow.h dmr_c/src/ident-list.h
+        dmr_c/src/linearize.h dmr_c/src/lib.h dmr_c/src/parse.h dmr_c/src/scope.h dmr_c/src/symbol.h
+        dmr_c/src/target.h dmr_c/src/token.h dmr_c/src/walksymbol.h)
 set(LUA_CMD_SRCS src/lua.c)
-set(RAVICOMP_SRCS src/ravi_complib.c)
 file(GLOB RAVI_HEADERS "${PROJECT_SOURCE_DIR}/include/*.h")
 
 if (COMPUTED_GOTO AND NOT MSVC)
@@ -61,35 +111,127 @@ endif ()
 include(CheckCCompilerFlag)
 check_c_compiler_flag("-march=native" COMPILER_OPT_ARCH_NATIVE_SUPPORTED)
 if (COMPILER_OPT_ARCH_NATIVE_SUPPORTED AND NOT CMAKE_C_FLAGS MATCHES "-march=")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native")
-endif ()
-if (ASAN)
-    set(CMAKE_REQUIRED_FLAGS "-fsanitize=address")
-    check_c_compiler_flag("-fsanitize=address" COMPILER_ASAN_SUPPORTED)
-    if (COMPILER_ASAN_SUPPORTED AND NOT CMAKE_C_FLAGS_DEBUG MATCHES "-fsanitize=address")
-        set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fsanitize=address")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native")
+endif()
+check_c_compiler_flag("-fno-common" COMPILER_OPT_NO_COMMON_SUPPORTED)
+if (COMPILER_OPT_NO_COMMON_SUPPORTED AND NOT CMAKE_C_FLAGS MATCHES "-fno-common")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-common")
+endif()
+check_c_compiler_flag("-fno-stack-protector" COMPILER_OPT_NO_STACK_PROTECTOR_SUPPORTED)
+if (COMPILER_OPT_NO_STACK_PROTECTOR_SUPPORTED AND NOT CMAKE_C_FLAGS MATCHES "-fno-stack-protector")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-stack-protector")
+endif()
+
+
+set(EMBEDDED_DMRC ON)
+if (LLVM_JIT)
+    find_package(LLVM REQUIRED CONFIG)
+    message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+    message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+
+    message(STATUS "LLVM Definitions ${LLVM_DEFINITIONS}")
+    message(STATUS "LLVMJIT enabled")
+
+    set(JIT_SRCS ${LLVM_JIT_SRCS})
+    if (NOT MSVC)
+        set_source_files_properties(${LLVM_JIT_SRCS} PROPERTIES
+                COMPILE_FLAGS "-fno-rtti -fno-exceptions ${LLVM_DEFINITIONS}")
     endif ()
-endif ()
+    set_property(
+            SOURCE ${LLVM_JIT_SRCS}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES ${LLVM_INCLUDE_DIRS}
+    )
+    set(DMR_C_JIT_SRCS
+            dmr_c/llvm-backend/sparse-llvm.c
+            )
+    set(DMR_C_JIT_HEADERS
+            dmr_c/llvm-backend/dmr_c.h
+            )
+    set(DMR_C_HEADERS ${DMR_C_HEADERS_OPT} ${DMR_C_HEADERS_COMMON})
+    set(DMR_C_SRCS ${DMR_C_SRCS_OPT} ${DMR_C_SRCS_COMMON} ${RAVI_C_SYMBOL_PARSER})
 
-if (MIR_JIT)
-    message(STATUS "MIRJIT enabled")
-    add_subdirectory(mir)
-    set(MIRJIT_LIBRARIES c2mir)
-    set(JIT_SRCS ${MIR_JIT_SRCS})
+    set_property(SOURCE ${DMR_C_JIT_SRCS} src/ravi_llvmcodegen.cpp ${RAVI_C_SYMBOL_PARSER}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/dmr_c/llvm-backend")
+    set_property(SOURCE ${DMR_C_SRCS} ${DMR_C_JIT_SRCS} ${RAVI_AST_SOURCES}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/dmr_c/src")
+    set_property(
+            SOURCE ${RAVI_C_SYMBOL_PARSER} ${DMR_C_JIT_SRCS}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES ${LLVM_INCLUDE_DIRS})
+    # FIXME get rid of this dependency
+    set_property(
+            SOURCE ${LUA_CMD_SRCS}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES ${LLVM_INCLUDE_DIRS})
+elseif (OMR_JIT)
+    find_package(OMRJIT REQUIRED)
+    message(STATUS "OMRJIT enabled")
+    set(JIT_SRCS ${OMR_JIT_SRCS})
+    set(DMR_C_JIT_SRCS
+            dmr_c/omrjit-backend/sparse-omrjit.c
+            )
+    set(DMR_C_JIT_HEADERS
+            dmr_c/omrjit-backend/dmr_c.h
+            )
+    set(DMR_C_HEADERS ${DMR_C_HEADERS_OPT} ${DMR_C_HEADERS_COMMON})
+    set(DMR_C_SRCS ${DMR_C_SRCS_OPT} ${DMR_C_SRCS_COMMON} ${RAVI_C_SYMBOL_PARSER})
+
+    set_property(SOURCE ${DMR_C_JIT_SRCS} ${RAVI_C_SYMBOL_PARSER} ${OMR_JIT_SRCS}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/dmr_c/omrjit-backend")
+    set_property(SOURCE ${DMR_C_SRCS} ${DMR_C_JIT_SRCS} ${RAVI_AST_SOURCES}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/dmr_c/src")
+    set_property(SOURCE ${DMR_C_JIT_SRCS} ${OMR_JIT_SRCS} ${RAVI_C_SYMBOL_PARSER}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES "${OMRJIT_INCLUDE_DIRS}"
+            )
 else ()
-    set(JIT_SRCS ${NO_JIT_SRCS})
+    # Omit all dmrC stuff except for what we need for the parser
+    set(DMR_C_HEADERS ${DMR_C_HEADERS_COMMON})
+    set(DMR_C_SRCS ${DMR_C_SRCS_COMMON})
+    set(EMBEDDED_DMRC OFF)
+
+    set_property(SOURCE ${DMR_C_SRCS_COMMON} ${RAVI_AST_SOURCES}
+            APPEND
+            PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/dmr_c/src")
+
+    if (MIR_JIT)
+        message(STATUS "MIRJIT enabled")
+        set(JIT_SRCS ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS})
+        set_property(SOURCE ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS}
+                APPEND
+                PROPERTY INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/mir;${CMAKE_SOURCE_DIR}/mir/c2mir")
+        set_property(SOURCE ${MIR_SRCS} ${C2MIR_SRCS} ${MIR_JIT_SRCS}
+                APPEND
+                PROPERTY COMPILE_DEFINITIONS "x86_64;MIR_IO;MIR_SCAN")
+    else()
+        set(JIT_SRCS ${NO_JIT_SRCS})
+    endif ()
 endif ()
 
-if (RAVICOMP)
-    set(ADDON_SRCS ${RAVICOMP_SRCS})
-    add_subdirectory(ravicomp)
-    set(RAVICOMP_LIBRARIES ravicomp)
+# Additional stuff for dmrC
+if (CMAKE_COMPILER_IS_GNUCC AND EMBEDDED_DMRC)
+    execute_process(COMMAND ${CMAKE_C_COMPILER} --print-file-name=
+            OUTPUT_VARIABLE GCC_BASE OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process(COMMAND ${CMAKE_C_COMPILER} -print-multiarch
+            OUTPUT_VARIABLE MULTIARCH_TRIPLET ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    set_source_files_properties(dmr_c/src/lib.c PROPERTIES COMPILE_FLAGS "-DGCC_BASE=${GCC_BASE} -DMULTIARCH_TRIPLET=${MULTIARCH_TRIPLET}")
+
+    message(STATUS "GCC_BASE_DIR      : " ${GCC_BASE})
+    message(STATUS "MULTIARCH_TRIPLET : " ${MULTIARCH_TRIPLET})
 endif ()
 
 # IDE stuff
 if (MSVC OR APPLE)
+    source_group("dmrC Headers" FILES ${DMR_C_HEADERS} ${DMR_C_JIT_HEADERS})
+    source_group("dmrC Source Files" FILES ${DMR_C_SRCS} ${DMR_C_JIT_SRCS})
     source_group("Ravi Headers" FILES ${RAVI_HEADERS})
-    source_group("Ravi Source Files" FILES ${LUA_CORE_SRCS} ${LUA_LIB_SRCS} ${JIT_SRCS} ${ADDON_SRCS})
+    source_group("Ravi Source Files" FILES ${LUA_CORE_SRCS} ${LUA_LIB_SRCS} ${JIT_SRCS})
 endif ()
 
 # Misc setup
@@ -104,6 +246,63 @@ elseif (NOT WIN32)
     set(EXTRA_LIBRARIES m dl readline)
 endif ()
 
+# Enable minimal required LLVM components so that the
+# the size of the resulting binary is manageable
+if (LLVM_JIT)
+    if (${LLVM_PACKAGE_VERSION} VERSION_LESS "3.8")
+        set(LLVM_EXTRA_LIBS ipa)
+    endif ()
+    if (NOT ${LLVM_PACKAGE_VERSION} VERSION_LESS "5.0.0")
+        set(LLVM_EXTRA_LIBS orcjit)
+    endif ()
+    message(STATUS "SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}")
+    if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+        set(LLVM_LIBS_PROCESSOR
+                X86CodeGen
+                X86AsmParser
+                X86Disassembler
+                X86Desc
+                X86Info
+                X86Utils
+                )
+        if (${LLVM_PACKAGE_VERSION} VERSION_LESS "9.0.0")
+            list(APPEND LLVM_LIBS_PROCESSOR X86AsmPrinter)
+        endif ()
+    elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
+        set(LLVM_LIBS_PROCESSOR
+                ARMCodeGen
+                ARMAsmParser
+                ARMDisassembler
+                ARMAsmPrinter
+                ARMDesc
+                ARMInfo
+                )
+    endif ()
+    llvm_map_components_to_libnames(LLVM_LIBS
+            Analysis
+            AsmParser
+            AsmPrinter
+            BitReader
+            Core
+            CodeGen
+            ExecutionEngine
+            InstCombine
+            ${LLVM_EXTRA_LIBS}
+            ipo
+            MC
+            MCJIT
+            MCParser
+            Object
+            RuntimeDyld
+            ScalarOpts
+            Support
+            Target
+            TransformUtils
+            ${LLVM_LIBS_PROCESSOR}
+            )
+    message(STATUS "LLVM_LIBS ${LLVM_LIBS}")
+endif ()
+
 set(LIBRAVI_NAME libravi)
 
 #Main library
@@ -112,8 +311,10 @@ add_library(${LIBRAVI_NAME} ${LIBRAVI_BUILD_TYPE}
         ${LUA_LIB_SRCS}
         ${LUA_CORE_SRCS}
         ${JIT_SRCS}
-        ${ADDON_SRCS})
-target_link_libraries(${LIBRAVI_NAME} LINK_PUBLIC ${EXTRA_LIBRARIES} ${MIRJIT_LIBRARIES} ${RAVICOMP_LIBRARIES})
+        ${DMR_C_HEADERS}
+        ${DMR_C_SRCS}
+        ${DMR_C_JIT_SRCS})
+target_link_libraries(${LIBRAVI_NAME} ${EXTRA_LIBRARIES} ${LLVM_LIBS} ${OMRJIT_LIBRARIES} ${MIRJIT_LIBRARIES})
 
 # Main Ravi executable
 add_executable(ravi ${LUA_CMD_SRCS})
@@ -124,6 +325,9 @@ set(NOJIT_RAVI_SRCS
         ${RAVI_HEADERS}
         ${LUA_LIB_SRCS}
         ${LUA_CORE_SRCS}
+        ${DMR_C_HEADERS}
+        ${DMR_C_SRCS}
+        ${DMR_C_JIT_SRCS}
         ${NO_JIT_SRCS})
 set(RAVI_STATICEXEC_TARGET ravi_s)
 
@@ -150,7 +354,19 @@ if (NOT LTESTS)
             PROPERTY COMPILE_DEFINITIONS NO_LUA_DEBUG)
     set(NO_LUA_DEBUG 1)
 endif ()
-if (MIR_JIT)
+if (LLVM_JIT)
+    set_property(
+            TARGET ${LIBRAVI_NAME} ravi
+            APPEND
+            PROPERTY COMPILE_DEFINITIONS "USE_LLVM=1")
+    set(USE_LLVM 1)
+elseif (OMR_JIT)
+    set_property(
+            TARGET ${LIBRAVI_NAME} ravi
+            APPEND
+            PROPERTY COMPILE_DEFINITIONS "USE_OMRJIT=1")
+    set(USE_OMRJIT 1)
+elseif (MIR_JIT)
     set_property(
             TARGET ${LIBRAVI_NAME} ravi
             APPEND
@@ -170,12 +386,11 @@ if (NOT STATIC_BUILD)
 else ()
     set_target_properties(${LIBRAVI_NAME} PROPERTIES PREFIX "")
 endif ()
-if (RAVICOMP)
+if (EMBEDDED_DMRC)
     set_property(
-            TARGET ${LIBRAVI_NAME}
+            TARGET ${LIBRAVI_NAME} ravi
             APPEND
-            PROPERTY COMPILE_DEFINITIONS "USE_RAVICOMP=1")
-    set(USE_RAVICOMP 1)
+            PROPERTY COMPILE_DEFINITIONS "USE_DMR_C=1")
 endif ()
 if (APPLE)
     set_property(
@@ -188,6 +403,8 @@ elseif (UNIX)
             APPEND
             PROPERTY COMPILE_DEFINITIONS "LUA_USE_LINUX=1")
 endif ()
+set_property(TARGET ${LIBRAVI_NAME} PROPERTY CXX_STANDARD 14)
+set_property(TARGET ${LIBRAVI_NAME} PROPERTY CXX_EXTENSIONS OFF)
 
 include(GNUInstallDirs)
 configure_file(ravi-config.h.in ravi-config.h @ONLY)

Makefile

@@ -6,7 +6,7 @@
 # == CHANGE THE SETTINGS BELOW TO SUIT YOUR ENVIRONMENT =======================
 
 # Your platform. See PLATS for possible values.
-PLAT= guess
+PLAT= none
 
 # Where to install. The installation starts in the src and doc directories,
 # so take care if INSTALL_TOP is not an absolute path. See the local target.
@@ -38,7 +38,7 @@ RM= rm -f
 # == END OF USER SETTINGS -- NO NEED TO CHANGE ANYTHING BELOW THIS LINE =======
 
 # Convenience platforms targets.
-PLATS= guess aix bsd c89 freebsd generic linux linux-noreadline macosx mingw posix solaris
+PLATS= aix bsd c89 freebsd generic linux macosx mingw posix solaris
 
 # What to install.
 TO_BIN= ravi

README.md

@@ -1,195 +0,0 @@
-Ravi Programming Language
-=========================
-
-![image](https://travis-ci.org/dibyendumajumdar/ravi.svg?branch=master%0A%20:target:%20https://travis-ci.org/dibyendumajumdar/ravi)
-
-Ravi is a dialect of [Lua](http://www.lua.org/) with limited optional
-static typing and features [MIR](https://github.com/vnmakarov/mir)
-powered JIT compilers. The name Ravi comes from the Sanskrit word for
-the Sun. Interestingly a precursor to Lua was
-[Sol](http://www.lua.org/history.html) which had support for static
-types; Sol means the Sun in Portugese.
-
-Lua is perfect as a small embeddable dynamic language so why a
-derivative? Ravi extends Lua with static typing for improved performance
-when JIT compilation is enabled. However, the static typing is optional
-and therefore Lua programs are also valid Ravi programs.
-
-There are other attempts to add static typing to Lua - e.g. [Typed
-Lua](https://github.com/andremm/typedlua) but these efforts are mostly
-about adding static type checks in the language while leaving the VM
-unmodified. The Typed Lua effort is very similar to the approach taken
-by Typescript in the JavaScript world. The static typing is to aid
-programming in the large - the code is eventually translated to standard
-Lua and executed in the unmodified Lua VM.
-
-My motivation is somewhat different - I want to enhance the VM to
-support more efficient operations when types are known. Type information
-can be exploited by JIT compilation technology to improve performance.
-At the same time, I want to keep the language safe and therefore usable
-by non-expert programmers.
-
-Of course there is the fantastic [LuaJIT](http://luajit.org)
-implementation. Ravi has a different goal compared to LuaJIT. Ravi
-prioritizes ease of maintenance and support, language safety, and
-compatibility with Lua 5.3, over maximum performance. For more detailed
-comparison please refer to the documentation links below.
-
-Features
---------
-
--   Optional static typing - for details [see the reference
-    manual](https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-reference.html).
--   Type specific bytecodes to improve performance
--   Compatibility with Lua 5.3 (see Compatibility section below)
--   Generational GC from Lua 5.4
--   `defer` statement for releasing resources
--   Compact JIT backend [MIR](https://github.com/vnmakarov/mir).
--   A [distribution with
-    batteries](https://github.com/dibyendumajumdar/Suravi).
--   A [Visual Studio Code debugger
-    extension](https://marketplace.visualstudio.com/items?itemName=ravilang.ravi-debug)
-    - interpreted mode debugger.
-
-Documentation
--------------
-
--   For the Lua extensions in Ravi see the [Reference
-    Manual](https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-reference.html).
--   [MIR JIT Build
-    instructions](https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-mir-instructions.html).
--   Also see [Ravi
-    Documentation](http://the-ravi-programming-language.readthedocs.org/en/latest/index.html).
--   and the slides I presented at the [Lua 2015
-    Workshop](http://www.lua.org/wshop15.html).
-
-Lua Goodies
------------
-
--   [An Introduction to
-    Lua](http://the-ravi-programming-language.readthedocs.io/en/latest/lua-introduction.html)
-    attempts to provide a quick overview of Lua for folks coming from
-    other languages.
--   [Lua 5.3 Bytecode
-    Reference](http://the-ravi-programming-language.readthedocs.io/en/latest/lua_bytecode_reference.html)
-    is my attempt to bring up to date the [Lua 5.1 Bytecode
-    Reference](http://luaforge.net/docman/83/98/ANoFrillsIntroToLua51VMInstructions.pdf).
--   A [patch for Lua
-    5.3](https://github.com/dibyendumajumdar/ravi/blob/master/patches/defer_statement_for_Lua_5_3.patch)
-    implements the 'defer' statement.
--   A [patch for Lua
-    5.4.[0-2]](https://github.com/dibyendumajumdar/ravi/blob/master/patches/defer_statement_for_Lua_5_4.patch)
-    implements the 'defer' statement.
--   Updated [patch for Lua
-    5.4.3](https://github.com/dibyendumajumdar/ravi/blob/master/patches/defer_statement_patch_for_Lua_5_4_3.patch)
-    implements the 'defer' statement.
-
-Lua 5.4 Position Statement
---------------------------
-
-Lua 5.4 relationship to Ravi is as follows:
-
--   Generational GC - back-ported to Ravi.
--   New random number generator - back-ported to Ravi.
--   Multiple user values can be associated with userdata - under
-    consideration.
--   `<const>` variables - not planned.
--   `<close>` variables - Ravi has `'defer'` statement which is the
-    better option in my opinion, hence no plans to support `<close>`
-    variables.
--   Interpreter performance improvements - these are beneficial to Lua
-    interpreter but not to the JIT backends, hence not much point in
-    back-porting.
--   Table implementation changes - under consideration.
--   String to number coercion is now part of string library metamethods
-    - back-ported to Ravi.
--   utf8 library accepts codepoints up to 2\^31 - back-ported to Ravi.
--   Removal of compatibility layers for 5.1, and 5.2 - not implemented
-    as Ravi continues to provide these layers as per Lua 5.3.
-
-Compatibility with Lua 5.3
---------------------------
-
-Ravi should be able to run all Lua 5.3 programs in interpreted mode, but
-following should be noted:
-
--   Ravi supports optional typing and enhanced types such as arrays (see
-    the documentation). Programs using these features cannot be run by
-    standard Lua. However all types in Ravi can be passed to Lua
-    functions; operations on Ravi arrays within Lua code will be subject
-    to restrictions as described in the section above on arrays.
--   Values crossing from Lua to Ravi will be subjected to typechecks
-    should these values be assigned to typed variables.
--   Upvalues cannot subvert the static typing of local variables (issue
-    \#26) when types are annotated.
--   Certain Lua limits are reduced due to changed byte code structure.
-    These are described below.
--   Ravi uses an extended bytecode which means it is not compatible with
-    Lua 5.x bytecode.
--   Ravi incorporates the new Generational GC from Lua 5.4, hence the GC
-    interface has changed.
-
-  Limit name         Lua value      Ravi value
-  ------------------ -------------- --------------
-  MAXUPVAL           255            125
-  LUAI\_MAXCCALLS    200            125
-  MAXREGS            255            125
-  MAXVARS            200            125
-  MAXARGLINE         250            120
-
-When JIT compilation is enabled there are following additional
-constraints:
-
--   Ravi will only execute JITed code from the main Lua thread; any
-    secondary threads (coroutines) execute in interpreter mode.
--   In JITed code tailcalls are implemented as regular calls so unlike
-    the interpreter VM which supports infinite tail recursion JIT
-    compiled code only supports tail recursion to a depth of about 110
-    (issue \#17)
--   Debug api and hooks are not supported in JIT mode
-
-History
--------
-
--   2015
-    :   -   Implemented JIT compilation using LLVM
-        -   Implemented [libgccjit based alternative
-            JIT](https://github.com/dibyendumajumdar/ravi/tree/gccjit-ravi534)
-            (now discontinued)
-
--   2016
-    :   -   Implemented debugger for Ravi and Lua 5.3 for [Visual Studio
-            Code](https://github.com/dibyendumajumdar/ravi/tree/master/vscode-debugger)
-
--   2017
-    :   -   Embedded C compiler using dmrC project (C JIT compiler) (now
-            discontinued)
-        -   Additional type-annotations
-
--   2018
-    :   -   Implemented [Eclipse OMR JIT
-            backend](https://github.com/dibyendumajumdar/ravi/tree/omrjit)
-            (now discontinued)
-        -   Created [Ravi with
-            batteries](https://github.com/dibyendumajumdar/Suravi).
-
--   2019
-    :   -   New language feature - defer statement
-        -   New JIT backend [MIR](https://github.com/vnmakarov/mir).
-
--   2020
-    :   -   [New parser / type checker /
-            compiler](https://github.com/dibyendumajumdar/ravi-compiler)
-        -   Generational GC back-ported from Lua 5.4
-        -   Support for [LLVM
-            backend](https://github.com/dibyendumajumdar/ravi/tree/llvm)
-            archived
-
--   2021 (Plan)
-    :   -   Integrated AOT and JIT compilation support
-        -   Ravi 1.0 release
-
-License
--------
-
-MIT License

README.rst

@@ -0,0 +1,109 @@
+=========================
+Ravi Programming Language
+=========================
+.. image:: https://travis-ci.org/dibyendumajumdar/ravi.svg?branch=master
+    :target: https://travis-ci.org/dibyendumajumdar/ravi
+
+Ravi is a derivative/dialect of `Lua 5.3 <http://www.lua.org/>`_ with limited optional static typing and 
+features `MIR <https://github.com/vnmakarov/mir>`_, `LLVM <http://www.llvm.org/>`_ and `Eclipse OMR <https://github.com/dibyendumajumdar/nj>`_ 
+powered JIT compilers. The name Ravi comes from the Sanskrit word for the Sun. 
+Interestingly a precursor to Lua was `Sol <http://www.lua.org/history.html>`_ which had support for 
+static types; Sol means the Sun in Portugese.
+
+Lua is perfect as a small embeddable dynamic language so why a derivative? Ravi extends Lua with 
+static typing for improved performance when JIT compilation is enabled. However, the static typing is 
+optional and therefore Lua programs are also valid Ravi programs.
+
+There are other attempts to add static typing to Lua - e.g. `Typed Lua <https://github.com/andremm/typedlua>`_ but 
+these efforts are mostly about adding static type checks in the language while leaving the VM unmodified. 
+The Typed Lua effort is very similar to the approach taken by Typescript in the JavaScript world. 
+The static typing is to aid programming in the large - the code is eventually translated to standard Lua 
+and executed in the unmodified Lua VM.
+
+My motivation is somewhat different - I want to enhance the VM to support more efficient operations when types are 
+known. Type information can be exploited by JIT compilation technology to improve performance. At the same time, 
+I want to keep the language safe and therefore usable by non-expert programmers. 
+
+Of course there is the fantastic `LuaJIT <http://luajit.org>`_ implementation. Ravi has a different goal compared to 
+LuaJIT. Ravi prioritizes ease of maintenance and support, language safety, and compatibility with Lua 5.3, 
+over maximum performance. For more detailed comparison please refer to the documentation links below.
+
+Features
+========
+* Optional static typing - for details `see the reference manual <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-reference.html>`_.
+* Type specific bytecodes to improve performance
+* Compatibility with Lua 5.3 (see Compatibility section below)
+* New! JIT backend `MIR <https://github.com/vnmakarov/mir>`_; only Linux and x86-64 supported for now.
+* `LLVM <http://www.llvm.org/>`_ powered JIT compiler
+* `Eclipse OMR <https://github.com/dibyendumajumdar/nj>`_ powered JIT compiler
+* A `distribution with batteries <https://github.com/dibyendumajumdar/Suravi>`_.
+
+Documentation
+=============
+* For the Lua extensions in Ravi see the `Reference Manual <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-reference.html>`_.
+* `MIR JIT Build instructions <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-mir-instructions.html>`_.
+* `OMR JIT Build instructions <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-omr-instructions.html>`_.
+* `LLVM JIT Build instructions <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-llvm-instructions.html>`_.
+* Also see `Ravi Documentation <http://the-ravi-programming-language.readthedocs.org/en/latest/index.html>`_.
+* and the slides I presented at the `Lua 2015 Workshop <http://www.lua.org/wshop15.html>`_.
+
+Lua Goodies
+===========
+* `An Introduction to Lua <http://the-ravi-programming-language.readthedocs.io/en/latest/lua-introduction.html>`_ attempts to provide a quick overview of Lua for folks coming from other languages.
+* `Lua 5.3 Bytecode Reference <http://the-ravi-programming-language.readthedocs.io/en/latest/lua_bytecode_reference.html>`_ is my attempt to bring up to date the `Lua 5.1 Bytecode Reference <http://luaforge.net/docman/83/98/ANoFrillsIntroToLua51VMInstructions.pdf>`_.
+* A `patch for Lua 5.3 <http://lua-users.org/lists/lua-l/2020-01/msg00004.html>`_ implements the 'defer' statement.
+
+Compatibility with Lua
+======================
+Ravi should be able to run all Lua 5.3 programs in interpreted mode, but following should be noted:
+
+* Ravi supports optional typing and enhanced types such as arrays (described above). Programs using these features cannot be run by standard Lua. However all types in Ravi can be passed to Lua functions; operations on Ravi arrays within Lua code will be subject to restrictions as described in the section above on arrays.
+* Values crossing from Lua to Ravi will be subjected to typechecks should these values be assigned to typed variables.
+* Upvalues cannot subvert the static typing of local variables (issue #26) when types are annotated.
+* Certain Lua limits are reduced due to changed byte code structure. These are described below.
+* Ravi uses an extended bytecode which means it is not compatible with Lua 5.3 bytecode.
+
++-----------------+-------------+-------------+
+| Limit name      | Lua value   | Ravi value  |
++=================+=============+=============+
+| MAXUPVAL        | 255         | 125         |
++-----------------+-------------+-------------+
+| LUAI_MAXCCALLS  | 200         | 125         |
++-----------------+-------------+-------------+
+| MAXREGS         | 255         | 125         |
++-----------------+-------------+-------------+
+| MAXVARS         | 200         | 125         |
++-----------------+-------------+-------------+
+| MAXARGLINE      | 250         | 120         |
++-----------------+-------------+-------------+
+
+When JIT compilation is enabled there are following additional constraints:
+
+* Ravi will only execute JITed code from the main Lua thread; any secondary threads (coroutines) execute in interpreter mode.
+* In JITed code tailcalls are implemented as regular calls so unlike the interpreter VM which supports infinite tail recursion JIT compiled code only supports tail recursion to a depth of about 110 (issue #17)
+
+History
+=======
+* 2015
+       - Implemented JIT compilation using LLVM
+       - Implemented libgccjit based alternative JIT (now discontinued)
+* 2016
+       - Implemented debugger for Ravi and Lua 5.3 for `Visual Studio Code <https://github.com/dibyendumajumdar/ravi/tree/master/vscode-debugger>`_
+* 2017
+       - Embedded C compiler using dmrC project (C JIT compiler)
+       - Additional type-annotations
+* 2018
+       - Implemented Eclipse OMR JIT backend
+       - Created `Ravi with batteries <https://github.com/dibyendumajumdar/Suravi>`_.
+* 2019 
+       - `New parser, type checker for Ravi <https://the-ravi-programming-language.readthedocs.io/en/latest/ravi-new-parser-codegenerator.html>`_ - work in progress
+       - New language feature - `defer` statement
+       - New JIT backend `MIR <https://github.com/vnmakarov/mir>`_. 
+
+* 2020 (Plan)
+       - `New optimizing byte code generator based on new parser / type checker <https://github.com/dibyendumajumdar/ravi-compiler>`_
+       - Ravi 1.0 release
+
+License
+=======
+MIT License

build-utils/README.rst

@@ -5,4 +5,4 @@ The scripts here are unsupported - these are just my personal
 build scripts.
 
 
-The debug builds enable 'ltests' and address sanitizer.
+The unix LLVM debug builds enable 'ltests' and address sanitizer.

build-utils/cmake-32bit-llvm-shared.bat

@@ -0,0 +1,5 @@
+mkdir llvm32
+cd llvm32
+rem cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM37_32\share\llvm\cmake -DBUILD_STATIC=OFF ..
+cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM39_32\lib\cmake\llvm -DSTATIC_BUILD=OFF ..
+cd ..

build-utils/cmake-32bit-llvm.bat

@@ -0,0 +1,5 @@
+mkdir llvm32
+cd llvm32
+rem cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM37_32\share\llvm\cmake ..
+cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM39\lib\cmake\llvm -DSTATIC_BUILD=ON ..
+cd ..

build-utils/cmake-32bit-llvmd.bat

@@ -0,0 +1,5 @@
+mkdir llvm32d
+cd llvm32d
+rem cmake -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM37_32\share\llvm\cmake ..
+cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=\d\ravi32 -G "Visual Studio 14" -DLLVM_JIT=ON -DLLVM_DIR=\d\LLVM39D_32\lib\cmake\llvm -DSTATIC_BUILD=ON ..
+cd ..

build-utils/cmake-64bit-llvm-debug.bat

@@ -0,0 +1,6 @@
+mkdir llvm64d
+cd llvm64d
+rem cmake -DCMAKE_INSTALL_PREFIX=c:\ravi64llvmd -G "Visual Studio 14 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\LLVM37debug\share\llvm\cmake ..
+rem cmake -DSTATIC_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=c:\d\ravi64llvmd -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DEMBEDDED_DMRC=ON -DLLVM_DIR=c:\d\LLVM39D64\lib\cmake\llvm ..
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm501d\lib\cmake\llvm ..
+cd ..

build-utils/cmake-64bit-llvm.bat

@@ -0,0 +1,7 @@
+mkdir llvm64
+cd llvm64
+rem pre LLVM 3.9
+rem cmake -DCMAKE_INSTALL_PREFIX=c:\ravi -G "Visual Studio 14 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\LLVM37\share\llvm\cmake ..
+rem cmake -DCMAKE_INSTALL_PREFIX=c:\ravi -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\d\LLVM40_64\lib\cmake\llvm ..
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DSTATIC_BUILD=ON -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm601r\lib\cmake\llvm ..
+cd ..

build-utils/cmake-64bit-llvm8.bat

@@ -0,0 +1,5 @@
+rmdir /s llvm8
+mkdir llvm8
+cd llvm8
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Release -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm801\lib\cmake\llvm ..
+cd ..

build-utils/cmake-64bit-llvm9d.bat

@@ -0,0 +1,5 @@
+rmdir /s llvm9d
+mkdir llvm9d
+cd llvm9d
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm900\lib\cmake\llvm ..
+cd ..

build-utils/cmake-64bit-llvm9r.bat

@@ -0,0 +1,6 @@
+rmdir /s llvm9r
+mkdir llvm9r
+cd llvm9r
+rem cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm900r\lib\cmake\llvm ..
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 16 2019" -DLLVM_JIT=ON -DLLVM_DIR=c:\Software\llvm900r\lib\cmake\llvm ..
+cd ..

build-utils/cmake-64bit-nojit-assertions.bat

@@ -1,4 +1,4 @@
 mkdir nojit64a
 cd nojit64a
-cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -DCMAKE_BUILD_TYPE=Debug -DLTESTS=ON  ..
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -DCMAKE_BUILD_TYPE=Debug -DLTESTS=ON -G "Visual Studio 15 2017 Win64" ..
 cd ..

build-utils/cmake-64bit-omrjit-debug.bat

@@ -0,0 +1,4 @@
+mkdir omrjit
+cd omrjit
+cmake -DCMAKE_INSTALL_PREFIX=c:\Software\ravi -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Debug -DOMR_JIT=ON ..
+cd ..

build-utils/cmake-64bit-osx-xcode-llvm-debug.sh

@@ -0,0 +1,4 @@
+mkdir xcodellvm
+cd xcodellvm
+#cmake -DCMAKE_BUILD_TYPE=Debug -G Xcode -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM/share/llvm/cmake ..
+cmake -DCMAKE_BUILD_TYPE=Debug -G Xcode -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM5/lib/cmake/llvm ..

build-utils/cmake-64bit-unix-llvm-debug.sh

@@ -0,0 +1,5 @@
+mkdir buildllvmd
+cd buildllvmd
+#cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM/share/llvm/cmake ..
+#cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DLTESTS=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM5/lib/cmake/llvm ..
+cmake -DCMAKE_BUILD_TYPE=Debug -DLLVM_JIT=ON -DLTESTS=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravillvm -DLLVM_DIR=$HOME/Software/llvm600/lib/cmake/llvm ..

build-utils/cmake-64bit-unix-llvm-release.sh

@@ -0,0 +1,6 @@
+rm -rf buildllvm
+mkdir buildllvm
+cd buildllvm
+#cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM/share/llvm/cmake ..
+#cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi -DLLVM_DIR=$HOME/LLVM5/lib/cmake/llvm ..
+cmake -DCMAKE_BUILD_TYPE=Release -DSTATIC_BUILD=ON -DLLVM_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/Software/ravi -DLLVM_DIR=$HOME/Software/llvm801/lib/cmake/llvm ..

build-utils/cmake-64bit-unix-omrjit-release.sh

@@ -0,0 +1,3 @@
+mkdir omrjit
+cd omrjit
+cmake -DCMAKE_BUILD_TYPE=Release -DOMR_JIT=ON -DCMAKE_INSTALL_PREFIX=$HOME/ravi ..

cmake/FindMIRJIT.cmake

@@ -0,0 +1,16 @@
+find_path(MIRJIT_INCLUDE_DIR c2mir.h
+  PATHS
+  c:/Software/mir/include/mir
+  ~/Software/mir/include/mir
+  NO_DEFAULT_PATH
+)
+
+find_library(MIRJIT_LIBRARY
+  NAMES libc2mir.a
+  PATHS
+  c:/Software/mir/lib
+  ~/Software/mir/lib
+)
+
+set( MIRJIT_INCLUDE_DIRS "${MIRJIT_INCLUDE_DIR}" )
+set( MIRJIT_LIBRARIES "${MIRJIT_LIBRARY}" )

cmake/FindOMRJIT.cmake

@@ -0,0 +1,16 @@
+find_path(OMRJIT_INCLUDE_DIR nj_api.h
+  PATHS
+  c:/Software/omr/include/nj
+  ~/Software/omr/include/nj
+  NO_DEFAULT_PATH
+)
+
+find_library(OMRJIT_LIBRARY
+  NAMES nj libnj libnj.dylib
+  PATHS
+  c:/Software/omr/lib
+  ~/Software/omr/lib
+)
+
+set( OMRJIT_INCLUDE_DIRS "${OMRJIT_INCLUDE_DIR}" )
+set( OMRJIT_LIBRARIES "${OMRJIT_LIBRARY}" )

dmr_c/README.md

@@ -0,0 +1,8 @@
+# dmr_C is a C Parser and JIT compiler
+
+Ravi includes a copy of the [dmr_C](https://github.com/dibyendumajumdar/dmr_c) project. See the project for more details regarding dmr_C.
+
+## Goals
+
+* Use dmr_C to translate code to JIT backend such as OMR JIT, NanoJIT or LLVM. 
+* Expose dmr_C features such as C parser, and compiler to users so that they can also compile chunks of C code when necessary

dmr_c/llvm-backend/dmr_c.h

@@ -0,0 +1,44 @@
+/**
+ * This is a backend code generator for dmr_C that uses
+ * the LLVM JIT engine.
+ *
+ * Copyright (C) 2017 Dibyendu Majumdar
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef DMR_C_H
+#define DMR_C_H
+
+#include <llvm-c/Core.h>
+
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern bool dmrC_llvmcompile(int argc, char **argv, LLVMModuleRef module,
+			     const char *inputbuffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

dmr_c/null-backend/dmr_c.h

@@ -0,0 +1,28 @@
+/**
+ * This header is used when no backend is used
+ *
+ * Copyright (C) 2017 Dibyendu Majumdar
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef DMR_C_H
+#define DMR_C_H
+
+#endif

dmr_c/omrjit-backend/dmr_c.h

@@ -0,0 +1,42 @@
+/**
+ * This is a backend code generator for dmr_C that uses
+ * the JIT engine OMR JIT (https://github.com/dibyendumajumdar/nj).
+ *
+ * Copyright (C) 2018 Dibyendu Majumdar
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef DMR_C_H
+#define DMR_C_H
+
+#include <nj_api.h>
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool dmrC_omrcompile(int argc, char **argv, JIT_ContextRef context,
+		      const char *inputbuffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

dmr_c/src/README.md

@@ -0,0 +1,67 @@
+The code is organised as follows:
+
+
+## Pointer Linked List
+
+This is implemented in:
+
+* ptrlist.h
+* ptrlist.c
+
+## Memory allocation
+
+* allocate.h
+* allocate.c
+
+## Tokenizer
+
+This is implemented in:
+
+* token.h
+* tokenize.c
+
+The tokenizer depends on declarations in symbol.h and target.h
+
+## Parser
+
+The C parser parses code as well as evaluates constant expressions and as far as I can tell it will also perform 
+inlining of functions. The evaluation code is reused by the pre-processor so the C parser is lower level than the 
+pre-processor.
+
+The parser implementation is in following:
+
+* symbol.h
+* symbol.c
+* scope.h
+* scope.c
+* parse.h
+* parse.c
+* char.c
+* char.h
+* expression.h
+* expression.c
+* expand.c
+* evaluate.c
+* inline.c
+* ident-list.h
+* show-parse.c
+
+## Pre-processor
+
+The pre-processor depends upon the parser functions to evaluate expressions. It is implemented in:
+
+* pre-process.c
+
+## Linearizer
+
+This component transforms the parsed representation to a linear form (SSA I believe). The implementation is in:
+
+* linearize.h
+* linearize.c
+* flow.h
+* flow.c
+* cse.c
+* liveness.c
+* memops.c 
+* simplify.c
+* unssa.c

dmr_c/src/allocate.c

@@ -0,0 +1,215 @@
+/*
+* allocate.c - simple space-efficient blob allocator.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003-2004 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*
+* Simple allocator for data that doesn't get partially free'd.
+* The tokenizer and parser allocate a _lot_ of small data structures
+* (often just two-three bytes for things like small integers),
+* and since they all depend on each other you can't free them
+* individually _anyway_. So do something that is very space-
+* efficient: allocate larger "blobs", and give out individual
+* small bits and pieces of it with no maintenance overhead.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <allocate.h>
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+void *dmrC_blob_alloc(size_t size) {
+  void *ptr;
+  ptr = malloc(size);
+  if (ptr != NULL)
+    memset(ptr, 0, size);
+  return ptr;
+}
+
+void dmrC_blob_free(void *addr, size_t size) {
+  (void)size;
+  free(addr);
+}
+
+void dmrC_allocator_init(struct allocator *A, const char *name, size_t size,
+                    unsigned int alignment, unsigned int chunking) {
+  A->name_ = name;
+  A->blobs_ = NULL;
+  A->size_ = size;
+  A->alignment_ = alignment;
+  A->chunking_ = chunking;
+  A->freelist_ = NULL;
+  A->allocations = 0;
+  A->total_bytes = 0;
+  A->useful_bytes = 0;
+}
+
+void *dmrC_allocator_allocate(struct allocator *A, size_t extra) {
+  size_t size = extra + A->size_;
+  size_t alignment = A->alignment_;
+  struct allocation_blob *blob = A->blobs_;
+  void *retval;
+
+  assert(size <= A->chunking_);
+  /*
+  * NOTE! The freelist only works with things that are
+  *  (a) sufficiently aligned
+  *  (b) use a constant size
+  * Don't try to free allocators that don't follow
+  * these rules.
+  */
+  if (A->freelist_) {
+    void **p = (void **)A->freelist_;
+    retval = p;
+    A->freelist_ = *p;
+    memset(retval, 0, size);
+    return retval;
+  }
+
+  A->allocations++;
+  A->useful_bytes += size;
+  size = (size + alignment - 1) & ~(alignment - 1);
+  if (!blob || blob->left < size) {
+    size_t offset, chunking = A->chunking_;
+    struct allocation_blob *newblob =
+        (struct allocation_blob *)dmrC_blob_alloc(chunking);
+    if (!newblob) {
+      fprintf(stderr, "out of memory\n");
+      abort();
+    }
+    A->total_bytes += chunking;
+    newblob->next = blob;
+    blob = newblob;
+    A->blobs_ = newblob;
+    offset = offsetof(struct allocation_blob, data);
+    offset = (offset + alignment - 1) & ~(alignment - 1);
+    blob->left = chunking - offset;
+    blob->offset = offset - offsetof(struct allocation_blob, data);
+  }
+  retval = blob->data + blob->offset;
+  blob->offset += size;
+  blob->left -= size;
+  return retval;
+}
+
+void dmrC_allocator_free(struct allocator *A, void *entry) {
+  void **p = (void **)entry;
+  *p = A->freelist_;
+  A->freelist_ = p;
+}
+void dmrC_allocator_show_allocations(struct allocator *A) {
+  fprintf(stderr, "%s: %d allocations, %d bytes (%d total bytes, "
+                  "%6.2f%% usage, %6.2f average size)\n",
+          A->name_, (int)A->allocations, (int)A->useful_bytes,
+          (int)A->total_bytes, 100 * (double)A->useful_bytes / A->total_bytes,
+          (double)A->useful_bytes / A->allocations);
+}
+void dmrC_allocator_drop_all_allocations(struct allocator *A) {
+  struct allocation_blob *blob = A->blobs_;
+  A->blobs_ = NULL;
+  A->allocations = 0;
+  A->total_bytes = 0;
+  A->useful_bytes = 0;
+  A->freelist_ = NULL;
+  while (blob) {
+    struct allocation_blob *next = blob->next;
+    dmrC_blob_free(blob, A->chunking_);
+    blob = next;
+  }
+}
+void dmrC_allocator_destroy(struct allocator *A) {
+  dmrC_allocator_drop_all_allocations(A);
+  A->blobs_ = NULL;
+  A->allocations = 0;
+  A->total_bytes = 0;
+  A->useful_bytes = 0;
+  A->freelist_ = NULL;
+}
+void dmrC_allocator_transfer(struct allocator *A, struct allocator *transfer_to) {
+  assert(transfer_to->blobs_ == NULL);
+  assert(transfer_to->freelist_ == NULL);
+  transfer_to->blobs_ = A->blobs_;
+  transfer_to->allocations = A->allocations;
+  transfer_to->total_bytes = A->total_bytes;
+  transfer_to->useful_bytes = A->useful_bytes;
+  transfer_to->freelist_ = A->freelist_;
+  transfer_to->alignment_ = A->alignment_;
+  transfer_to->chunking_ = A->chunking_;
+  transfer_to->size_ = A->size_;
+  A->blobs_ = NULL;
+  A->allocations = 0;
+  A->total_bytes = 0;
+  A->useful_bytes = 0;
+  A->freelist_ = NULL;
+}
+
+struct foo {
+  int a, b;
+};
+
+int dmrC_test_allocator() {
+  struct allocator alloc;
+  dmrC_allocator_init(&alloc, "foo", sizeof(struct foo), __alignof__(struct foo),
+                 sizeof(struct allocation_blob) + sizeof(struct foo) * 2);
+  struct foo *t1 = (struct foo *)dmrC_allocator_allocate(&alloc, 0);
+  if (t1 == NULL)
+    return 1;
+  if (alloc.alignment_ != __alignof__(struct foo))
+    return 1;
+  if (alloc.allocations != 1)
+    return 1;
+  if (alloc.freelist_ != NULL)
+    return 1;
+  struct foo *t2 = (struct foo *)dmrC_allocator_allocate(&alloc, 0);
+  if (t2 != t1 + 1)
+    return 1;
+  //dmrC_allocator_show_allocations(&alloc);
+  dmrC_allocator_free(&alloc, t1);
+  dmrC_allocator_free(&alloc, t2);
+  struct foo *t3 = (struct foo *)dmrC_allocator_allocate(&alloc, 0);
+  if (t3 != t2)
+    return 1;
+  struct foo *t4 = (struct foo *)dmrC_allocator_allocate(&alloc, 0);
+  if (t4 != t1)
+    return 1;
+  struct foo *t5 = (struct foo *)dmrC_allocator_allocate(&alloc, 0);
+  (void)t5;
+  if (alloc.total_bytes !=
+      (sizeof(struct allocation_blob) + sizeof(struct foo) * 2) * 2)
+    return 1;
+  struct allocator alloc2;
+  memset(&alloc2, 0, sizeof alloc2);
+  struct allocation_blob *saved = alloc.blobs_;
+  dmrC_allocator_transfer(&alloc, &alloc2);
+  if (alloc.blobs_ != NULL)
+	  return 1;
+  if (alloc2.blobs_ != saved)
+	  return 1;
+  dmrC_allocator_destroy(&alloc2);
+  printf("allocator tests okay\n");
+  return 0;
+}

dmr_c/src/allocate.h

@@ -0,0 +1,100 @@
+#ifndef DMR_C_ALLOCATOR_H
+#define DMR_C_ALLOCATOR_H
+
+/*
+* allocate.c - simple space-efficient blob allocator.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003-2004 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*
+* Simple allocator for data that doesn't get partially free'd.
+* The tokenizer and parser allocate a _lot_ of small data structures
+* (often just two-three bytes for things like small integers),
+* and since they all depend on each other you can't free them
+* individually _anyway_. So do something that is very space-
+* efficient: allocate larger "blobs", and give out individual
+* small bits and pieces of it with no maintenance overhead.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <port.h>
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct allocation_blob {
+	struct allocation_blob *next;
+	size_t left, offset;
+	unsigned char data[];
+};
+
+/*
+* Our "blob" allocator works on chunks that are multiples
+* of this size (the underlying allocator may be a mmap that
+* cannot handle smaller chunks, for example, so trying to
+* allocate blobs that aren't aligned is not going to work).
+*/
+#define CHUNK 32768
+
+struct allocator {
+	const char *name_;
+	struct allocation_blob *blobs_;
+	size_t size_;
+	unsigned int alignment_;
+	unsigned int chunking_;
+	void *freelist_;
+	size_t allocations, total_bytes, useful_bytes;
+};
+
+extern void dmrC_allocator_init(struct allocator *A, const char *name, size_t size,
+			   unsigned int alignment, unsigned int chunking);
+
+extern void *dmrC_allocator_allocate(struct allocator *A, size_t extra);
+
+extern void dmrC_allocator_free(struct allocator *A, void *entry);
+
+extern void dmrC_allocator_show_allocations(struct allocator *A);
+
+extern void dmrC_allocator_drop_all_allocations(struct allocator *A);
+
+extern void dmrC_allocator_destroy(struct allocator *A);
+
+extern void dmrC_allocator_transfer(struct allocator *A,
+			       struct allocator *transfer_to);
+
+extern int dmrC_test_allocator();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/builtin.c

@@ -0,0 +1,249 @@
+/*
+ * builtin evaluation & expansion.
+ *
+ * Copyright (C) 2003 Transmeta Corp.
+ *               2003-2004 Linus Torvalds
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <port.h>
+#include <lib.h>
+#include <expression.h>
+#include <expand.h>
+#include <symbol.h>
+
+static int evaluate_to_integer(struct dmr_C *C, struct expression *expr)
+{
+	expr->ctype = &C->S->int_ctype;
+	return 1;
+}
+
+static int evaluate_expect(struct dmr_C *C, struct expression *expr)
+{
+	/* Should we evaluate it to return the type of the first argument? */
+	expr->ctype = &C->S->int_ctype;
+	return 1;
+}
+
+static int arguments_choose(struct dmr_C *C, struct expression *expr)
+{
+	struct expression_list *arglist = expr->args;
+	struct expression *arg;
+	int i = 0;
+
+	FOR_EACH_PTR (arglist, arg) {
+		if (!dmrC_evaluate_expression(C, arg))
+			return 0;
+		i++;
+	} END_FOR_EACH_PTR(arg);
+	if (i < 3) {
+		dmrC_sparse_error(C, expr->pos,
+			     "not enough arguments for __builtin_choose_expr");
+		return 0;
+	} if (i > 3) {
+		dmrC_sparse_error(C, expr->pos,
+			     "too many arguments for __builtin_choose_expr");
+		return 0;
+	}
+	return 1;
+}
+
+static int evaluate_choose(struct dmr_C *C, struct expression *expr)
+{
+	struct expression_list *list = expr->args;
+	struct expression *arg, *args[3];
+	int n = 0;
+
+	/* there will be exactly 3; we'd already verified that */
+	FOR_EACH_PTR(list, arg) {
+		args[n++] = arg;
+	} END_FOR_EACH_PTR(arg);
+
+	*expr = dmrC_get_expression_value(C, args[0]) ? *args[1] : *args[2];
+
+	return 1;
+}
+
+static int expand_expect(struct dmr_C *C, struct expression *expr, int cost)
+{
+	struct expression *arg = dmrC_first_expression(expr->args);
+	(void)C;
+	(void)cost;
+
+	if (arg)
+		*expr = *arg;
+	return 0;
+}
+
+/*
+ * __builtin_warning() has type "int" and always returns 1,
+ * so that you can use it in conditionals or whatever
+ */
+static int expand_warning(struct dmr_C *C, struct expression *expr, int cost)
+{
+	struct expression *arg;
+	struct expression_list *arglist = expr->args;
+	(void)cost;
+
+	FOR_EACH_PTR (arglist, arg) {
+		/*
+		 * Constant strings get printed out as a warning. By the
+		 * time we get here, the EXPR_STRING has been fully 
+		 * evaluated, so by now it's an anonymous symbol with a
+		 * string initializer.
+		 *
+		 * Just for the heck of it, allow any constant string
+		 * symbol.
+		 */
+		if (arg->type == EXPR_SYMBOL) {
+			struct symbol *sym = arg->symbol;
+			if (sym->initializer && sym->initializer->type == EXPR_STRING) {
+				struct string *string = sym->initializer->string;
+				dmrC_warning(C, expr->pos, "%*s", string->length-1, string->data);
+			}
+			continue;
+		}
+
+		/*
+		 * Any other argument is a conditional. If it's
+		 * non-constant, or it is false, we exit and do
+		 * not print any warning.
+		 */
+		if (arg->type != EXPR_VALUE)
+			goto out;
+		if (!arg->value)
+			goto out;
+	} END_FOR_EACH_PTR(arg);
+out:
+	expr->type = EXPR_VALUE;
+	expr->value = 1;
+	expr->taint = 0;
+	return 0;
+}
+
+/* The arguments are constant if the cost of all of them is zero */
+static int expand_constant_p(struct dmr_C *C, struct expression *expr, int cost)
+{
+	(void) C;
+	expr->type = EXPR_VALUE;
+	expr->value = !cost;
+	expr->taint = 0;
+	return 0;
+}
+
+/* The arguments are safe, if their cost is less than SIDE_EFFECTS */
+static int expand_safe_p(struct dmr_C *C, struct expression *expr, int cost)
+{
+	(void) C;
+	expr->type = EXPR_VALUE;
+	expr->value = (cost < SIDE_EFFECTS);
+	expr->taint = 0;
+	return 0;
+}
+
+static struct symbol_op constant_p_op = {
+	.evaluate = evaluate_to_integer,
+	.expand = expand_constant_p
+};
+
+static struct symbol_op safe_p_op = {
+	.evaluate = evaluate_to_integer,
+	.expand = expand_safe_p
+};
+
+static struct symbol_op warning_op = {
+	.evaluate = evaluate_to_integer,
+	.expand = expand_warning
+};
+
+static struct symbol_op expect_op = {
+	.evaluate = evaluate_expect,
+	.expand = expand_expect
+};
+
+static struct symbol_op choose_op = {
+	.evaluate = evaluate_choose,
+	.args = arguments_choose,
+};
+
+/* The argument is constant and valid if the cost is zero */
+static int expand_bswap(struct dmr_C *C, struct expression *expr, int cost)
+{
+	struct expression *arg;
+	long long val;
+
+	if (cost)
+		return cost;
+
+	/* the arguments number & type have already been checked */
+	arg = dmrC_first_expression(expr->args);
+	val = dmrC_get_expression_value_silent(C, arg);
+	switch (expr->ctype->bit_size) {
+	case 16: expr->value = __builtin_bswap16((uint16_t)val); break;
+	case 32: expr->value = __builtin_bswap32((uint32_t)val); break;
+	case 64: expr->value = __builtin_bswap64(val); break;
+	default: /* impossible error */
+		return SIDE_EFFECTS;
+	}
+
+	expr->type = EXPR_VALUE;
+	expr->taint = 0;
+	return 0;
+}
+
+static struct symbol_op bswap_op = {
+	.expand = expand_bswap,
+};
+
+
+/*
+ * Builtin functions
+ */
+static struct symbol builtin_fn_type = { .type = SYM_FN /* , .variadic =1 */ };
+static struct sym_init {
+	const char *name;
+	struct symbol *base_type;
+	unsigned int modifiers;
+	struct symbol_op *op;
+} builtins_table[] = {
+	{ "__builtin_constant_p", &builtin_fn_type, MOD_TOPLEVEL, &constant_p_op },
+	{ "__builtin_safe_p", &builtin_fn_type, MOD_TOPLEVEL, &safe_p_op },
+	{ "__builtin_warning", &builtin_fn_type, MOD_TOPLEVEL, &warning_op },
+	{ "__builtin_expect", &builtin_fn_type, MOD_TOPLEVEL, &expect_op },
+	{ "__builtin_choose_expr", &builtin_fn_type, MOD_TOPLEVEL, &choose_op },
+	{ "__builtin_bswap16", NULL, MOD_TOPLEVEL, &bswap_op },
+	{ "__builtin_bswap32", NULL, MOD_TOPLEVEL, &bswap_op },
+	{ "__builtin_bswap64", NULL, MOD_TOPLEVEL, &bswap_op },
+	{ NULL,		NULL,		0, NULL }
+};
+
+void dmrC_init_builtins(struct dmr_C *C, int stream)
+{
+	struct sym_init *ptr;
+
+	builtin_fn_type.variadic = 1;
+	for (ptr = builtins_table; ptr->name; ptr++) {
+		struct symbol *sym;
+		sym = dmrC_create_symbol(C->S, stream, ptr->name, SYM_NODE, NS_SYMBOL);
+		sym->ctype.base_type = ptr->base_type;
+		sym->ctype.modifiers = ptr->modifiers;
+		sym->op = ptr->op;
+	}
+}

dmr_c/src/char.c

@@ -0,0 +1,181 @@
+/*
+* sparse/char.c
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003-2004 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+#include <string.h>
+
+#include <port.h>
+#include <target.h>
+#include <lib.h>
+#include <allocate.h>
+#include <token.h>
+#include <expression.h>
+
+static const char *parse_escape(struct dmr_C *C, const char *p, unsigned *val, const char *end, int bits, struct position pos)
+{
+	unsigned c = *p++;
+	unsigned d;
+	if (c != '\\') {
+		*val = c;
+		return p;
+	}
+
+	c = *p++;
+	switch (c) {
+	case 'a': c = '\a'; break;
+	case 'b': c = '\b'; break;
+	case 't': c = '\t'; break;
+	case 'n': c = '\n'; break;
+	case 'v': c = '\v'; break;
+	case 'f': c = '\f'; break;
+	case 'r': c = '\r'; break;
+#ifndef _MSC_VER
+	case 'e': c = '\e'; break;
+#endif
+	case 'x': {
+		unsigned mask = -(1U << (bits - 4));
+		for (c = 0; p < end; c = (c << 4) + d) {
+			d = dmrC_hexval(*p);
+			if (d > 16)
+				break;
+			p++;
+			if (c & mask) {
+				dmrC_warning(C, pos,
+					"hex escape sequence out of range");
+				mask = 0;
+			}
+		}
+		break;
+	}
+	case '0':case '1': case '2': case '3': case '4': case '5':
+	case '6': case '7': {
+		if (p + 2 < end)
+			end = p + 2;
+		c -= '0';
+		while (p < end && (d = *p - '0') < 8) {
+			c = (c << 3) + d;
+			p++;
+		}
+		if ((c & 0400) && bits < 9)
+			dmrC_warning(C, pos,
+				"octal escape sequence out of range");
+		break;
+	}
+	default:	/* everything else is left as is */
+		dmrC_warning(C, pos, "unknown escape sequence: '\\%c'", c);
+		break;
+	case '\\':
+	case '\'':
+	case '"':
+	case '?':
+		break;	/* those are legal, so no warnings */
+	}
+	*val = c & ~((~0U << (bits - 1)) << 1);
+	return p;
+}
+
+void dmrC_get_char_constant(struct dmr_C *C, struct token *token, unsigned long long *val)
+{
+	const char *p = token->embedded, *end;
+	unsigned v;
+	int type = dmrC_token_type(token);
+	switch (type) {
+	case TOKEN_CHAR:
+	case TOKEN_WIDE_CHAR:
+		p = token->string->data;
+		end = p + token->string->length - 1;
+		break;
+	case TOKEN_CHAR_EMBEDDED_0:
+	case TOKEN_CHAR_EMBEDDED_1:
+	case TOKEN_CHAR_EMBEDDED_2:
+	case TOKEN_CHAR_EMBEDDED_3:
+		end = p + type - TOKEN_CHAR;
+		break;
+	default:
+		end = p + type - TOKEN_WIDE_CHAR;
+	}
+	p = parse_escape(C, p, &v, end,
+			type < TOKEN_WIDE_CHAR ? C->target->bits_in_char : C->target->bits_in_wchar, token->pos);
+	if (p != end)
+		dmrC_warning(C, token->pos,
+			"multi-character character constant");
+	*val = v;
+}
+
+struct token *dmrC_get_string_constant(struct dmr_C *C, struct token *token, struct expression *expr)
+{
+	struct string *string = token->string;
+	struct token *next = token->next, *done = NULL;
+	int stringtype = dmrC_token_type(token);
+	int is_wide = stringtype == TOKEN_WIDE_STRING;
+	char buffer[MAX_STRING];
+	int len = 0;
+	int bits;
+	int esc_count = 0;
+
+	while (!done) {
+		switch (dmrC_token_type(next)) {
+		case TOKEN_WIDE_STRING:
+			is_wide = 1;
+		case TOKEN_STRING:
+			next = next->next;
+			break;
+		default:
+			done = next;
+		}
+	}
+	bits = is_wide ? C->target->bits_in_wchar : C->target->bits_in_char;
+	while (token != done) {
+		unsigned v;
+		const char *p = token->string->data;
+		const char *end = p + token->string->length - 1;
+		while (p < end) {
+			if (*p == '\\')
+				esc_count++;
+			p = parse_escape(C, p, &v, end, bits, token->pos);
+			if (len < MAX_STRING)
+				buffer[len] = v;
+			len++;
+		}
+		token = token->next;
+	}
+	if (len > MAX_STRING) {
+		dmrC_warning(C, token->pos, "trying to concatenate %d-character string (%d bytes max)", len, MAX_STRING);
+		len = MAX_STRING;
+	}
+
+	if (esc_count || len >= (int)string->length) {
+		if (string->immutable || len >= string->length)	/* can't cannibalize */
+			string = (struct string *)dmrC_allocator_allocate(&C->string_allocator, len+1);
+		string->length = len+1;
+		memcpy(string->data, buffer, len);
+		string->data[len] = '\0';
+	}
+	expr->string = string;
+	expr->wide = is_wide;
+	return token;
+}

dmr_c/src/char.h

@@ -0,0 +1,25 @@
+/*
+* sparse/char.h
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+#ifndef DMR_C_CHAR_H
+#define DMR_C_CHAR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern void dmrC_get_char_constant(struct dmr_C *C, struct token *, unsigned long long *);
+extern struct token *dmrC_get_string_constant(struct dmr_C *C, struct token *, struct expression *);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/cse.c

@@ -0,0 +1,393 @@
+/*
+ * CSE - walk the linearized instruction flow, and
+ * see if we can simplify it and apply CSE on it.
+ *
+ * Copyright (C) 2004 Linus Torvalds
+ */
+
+#include <string.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <assert.h>
+
+#include <port.h>
+#include <parse.h>
+#include <expression.h>
+#include <linearize.h>
+#include <flow.h>
+
+static int phi_compare(pseudo_t phi1, pseudo_t phi2)
+{
+	const struct instruction *def1 = phi1->def;
+	const struct instruction *def2 = phi2->def;
+
+	if (def1->src1 != def2->src1)
+		return def1->src1 < def2->src1 ? -1 : 1;
+	if (def1->bb != def2->bb)
+		return def1->bb < def2->bb ? -1 : 1;
+	return 0;
+}
+
+
+static void clean_up_one_instruction(struct dmr_C *C, struct basic_block *bb, struct instruction *insn)
+{
+	unsigned long hash;
+
+	if (!insn->bb)
+		return;
+	assert(insn->bb == bb);
+	C->L->repeat_phase |= dmrC_simplify_instruction(C, insn);
+	if (!insn->bb)
+		return;
+	hash = (insn->opcode << 3) + (insn->size >> 3);
+	switch (insn->opcode) {
+	case OP_SEL:
+		hash += dmrC_hashval(insn->src3);
+		/* Fall through */	
+
+	/* Binary arithmetic */
+	case OP_ADD: case OP_SUB:
+	case OP_MULU: case OP_MULS:
+	case OP_DIVU: case OP_DIVS:
+	case OP_MODU: case OP_MODS:
+	case OP_SHL:
+	case OP_LSR: case OP_ASR:
+	case OP_AND: case OP_OR:
+
+	/* Binary logical */
+	case OP_XOR: case OP_AND_BOOL:
+	case OP_OR_BOOL:
+
+	/* Binary comparison */
+	case OP_SET_EQ: case OP_SET_NE:
+	case OP_SET_LE: case OP_SET_GE:
+	case OP_SET_LT: case OP_SET_GT:
+	case OP_SET_B:  case OP_SET_A:
+	case OP_SET_BE: case OP_SET_AE:
+		hash += dmrC_hashval(insn->src2);
+		/* Fall through */
+	
+	/* Unary */
+	case OP_NOT: case OP_NEG:
+		hash += dmrC_hashval(insn->src1);
+		break;
+
+	case OP_SETVAL:
+		hash += dmrC_hashval(insn->val);
+		break;
+
+	case OP_SYMADDR:
+		hash += dmrC_hashval(insn->symbol);
+		break;
+
+	case OP_CAST:
+	case OP_SCAST:
+	case OP_PTRCAST:
+		/*
+		 * This is crap! Many "orig_types" are the
+		 * same as far as casts go, we should generate
+		 * some kind of "type hash" that is identical
+		 * for identical casts
+		 */
+		hash += dmrC_hashval(insn->orig_type);
+		hash += dmrC_hashval(insn->src);
+		break;
+
+	/* Other */
+	case OP_PHI: {
+		pseudo_t phi;
+		FOR_EACH_PTR(insn->phi_list, phi) {
+			struct instruction *def;
+			if (phi == VOID_PSEUDO(C) || !phi->def)
+				continue;
+			def = phi->def;
+			hash += dmrC_hashval(def->src1);
+			hash += dmrC_hashval(def->bb);
+		} END_FOR_EACH_PTR(phi);
+		break;
+	}
+
+	default:
+		/*
+		 * Nothing to do, don't even bother hashing them,
+		 * we're not going to try to CSE them
+		 */
+		return;
+	}
+	hash += hash >> 16;
+	hash &= INSN_HASH_SIZE-1;
+	dmrC_add_instruction(C, C->L->insn_hash_table + hash, insn);
+}
+
+static void clean_up_insns(struct dmr_C *C, struct entrypoint *ep)
+{
+	struct basic_block *bb;
+
+	FOR_EACH_PTR(ep->bbs, bb) {
+		struct instruction *insn;
+		FOR_EACH_PTR(bb->insns, insn) {
+			clean_up_one_instruction(C, bb, insn);
+		} END_FOR_EACH_PTR(insn);
+	} END_FOR_EACH_PTR(bb);
+}
+
+/* Compare two (sorted) phi-lists */
+static int phi_list_compare(struct dmr_C *C, struct pseudo_list *l1, struct pseudo_list *l2)
+{
+	pseudo_t phi1, phi2;
+
+	PREPARE_PTR_LIST(l1, phi1);
+	PREPARE_PTR_LIST(l2, phi2);
+	for (;;) {
+		int cmp;
+
+		while (phi1 && (phi1 == VOID_PSEUDO(C) || !phi1->def))
+			NEXT_PTR_LIST(phi1);
+		while (phi2 && (phi2 == VOID_PSEUDO(C) || !phi2->def))
+			NEXT_PTR_LIST(phi2);
+
+		if (!phi1)
+			return phi2 ? -1 : 0;
+		if (!phi2)
+			return phi1 ? 1 : 0;
+		cmp = phi_compare(phi1, phi2);
+		if (cmp)
+			return cmp;
+		NEXT_PTR_LIST(phi1);
+		NEXT_PTR_LIST(phi2);
+	}
+	/* Not reached, but we need to make the nesting come out right */
+	FINISH_PTR_LIST(phi2);
+	FINISH_PTR_LIST(phi1);
+}
+
+static int insn_compare(void *ud, const void *_i1, const void *_i2)
+{
+	struct dmr_C *C = (struct dmr_C *) ud;
+	const struct instruction *i1 = _i1;
+	const struct instruction *i2 = _i2;
+
+	if (i1->opcode != i2->opcode)
+		return i1->opcode < i2->opcode ? -1 : 1;
+
+	switch (i1->opcode) {
+	/* commutative binop */
+	case OP_ADD:
+	case OP_MULU: case OP_MULS:
+	case OP_AND_BOOL: case OP_OR_BOOL:
+	case OP_AND: case OP_OR:
+	case OP_XOR:
+	case OP_SET_EQ: case OP_SET_NE:
+		if (i1->src1 == i2->src2 && i1->src2 == i2->src1)
+			return 0;
+		goto case_binops;
+
+	case OP_SEL:
+		if (i1->src3 != i2->src3)
+			return i1->src3 < i2->src3 ? -1 : 1;
+		/* Fall-through to binops */
+
+	/* Binary arithmetic */
+	case OP_SUB:
+	case OP_DIVU: case OP_DIVS:
+	case OP_MODU: case OP_MODS:
+	case OP_SHL:
+	case OP_LSR: case OP_ASR:
+
+	/* Binary comparison */
+	case OP_SET_LE: case OP_SET_GE:
+	case OP_SET_LT: case OP_SET_GT:
+	case OP_SET_B:  case OP_SET_A:
+	case OP_SET_BE: case OP_SET_AE:
+	case_binops:
+		if (i1->src2 != i2->src2)
+			return i1->src2 < i2->src2 ? -1 : 1;
+		/* Fall through to unops */
+
+	/* Unary */
+	case OP_NOT: case OP_NEG:
+		if (i1->src1 != i2->src1)
+			return i1->src1 < i2->src1 ? -1 : 1;
+		break;
+
+	case OP_SYMADDR:
+		if (i1->symbol != i2->symbol)
+			return i1->symbol < i2->symbol ? -1 : 1;
+		break;
+
+	case OP_SETVAL:
+		if (i1->val != i2->val)
+			return i1->val < i2->val ? -1 : 1;
+		break;
+
+	/* Other */
+	case OP_PHI:
+		return phi_list_compare(C, i1->phi_list, i2->phi_list);
+
+	case OP_CAST:
+	case OP_SCAST:
+	case OP_PTRCAST:
+		/*
+		 * This is crap! See the comments on hashing.
+		 */
+		if (i1->orig_type != i2->orig_type)
+			return i1->orig_type < i2->orig_type ? -1 : 1;
+		if (i1->src != i2->src)
+			return i1->src < i2->src ? -1 : 1;
+		break;
+
+	default:
+		dmrC_warning(C, i1->pos, "bad instruction on hash chain");
+	}
+	if (i1->size != i2->size)
+		return i1->size < i2->size ? -1 : 1;
+	return 0;
+}
+
+static void sort_instruction_list(struct dmr_C *C, struct instruction_list **list)
+{
+	ptrlist_sort((struct ptr_list **)list, C, insn_compare);
+}
+
+static struct instruction * cse_one_instruction(struct dmr_C *C, struct instruction *insn, struct instruction *def)
+{
+	dmrC_convert_instruction_target(C, insn, def->target);
+
+	dmrC_kill_instruction(C, insn);
+	C->L->repeat_phase |= REPEAT_CSE;
+	return def;
+}
+
+/*
+ * Does "bb1" dominate "bb2"?
+ */
+static int bb_dominates(struct entrypoint *ep, struct basic_block *bb1, struct basic_block *bb2, unsigned long generation)
+{
+	struct basic_block *parent;
+
+	/* Nothing dominates the entrypoint.. */
+	if (bb2 == ep->entry->bb)
+		return 0;
+	FOR_EACH_PTR(bb2->parents, parent) {
+		if (parent == bb1)
+			continue;
+		if (parent->generation == generation)
+			continue;
+		parent->generation = generation;
+		if (!bb_dominates(ep, bb1, parent, generation))
+			return 0;
+	} END_FOR_EACH_PTR(parent);
+	return 1;
+}
+
+static struct basic_block *trivial_common_parent(struct basic_block *bb1, struct basic_block *bb2)
+{
+	struct basic_block *parent;
+
+	if (dmrC_bb_list_size(bb1->parents) != 1)
+		return NULL;
+	parent = dmrC_first_basic_block(bb1->parents);
+	if (dmrC_bb_list_size(bb2->parents) != 1)
+		return NULL;
+	if (dmrC_first_basic_block(bb2->parents) != parent)
+		return NULL;
+	return parent;
+}
+
+static inline void remove_instruction(struct instruction_list **list, struct instruction *insn, int count)
+{
+	ptrlist_remove((struct ptr_list **)list, insn, count);
+}
+
+static void add_instruction_to_end(struct dmr_C *C, struct instruction *insn, struct basic_block *bb)
+{
+	struct instruction *br = dmrC_delete_last_instruction(&bb->insns);
+	insn->bb = bb;
+	dmrC_add_instruction(C, &bb->insns, insn);
+	dmrC_add_instruction(C, &bb->insns, br);
+}
+
+static struct instruction * try_to_cse(struct dmr_C *C, struct entrypoint *ep, struct instruction *i1, struct instruction *i2)
+{
+	struct basic_block *b1, *b2, *common;
+
+	/*
+	 * OK, i1 and i2 are the same instruction, modulo "target".
+	 * We should now see if we can combine them.
+	 */
+	b1 = i1->bb;
+	b2 = i2->bb;
+
+	/*
+	 * Currently we only handle the uninteresting degenerate case where
+	 * the CSE is inside one basic-block.
+	 */
+	if (b1 == b2) {
+		struct instruction *insn;
+		FOR_EACH_PTR(b1->insns, insn) {
+			if (insn == i1)
+				return cse_one_instruction(C, i2, i1);
+			if (insn == i2)
+				return cse_one_instruction(C, i1, i2);
+		} END_FOR_EACH_PTR(insn);
+		dmrC_warning(C, b1->pos, "Whaa? unable to find CSE instructions");
+		return i1;
+	}
+	if (bb_dominates(ep, b1, b2, ++C->L->bb_generation))
+		return cse_one_instruction(C, i2, i1);
+
+	if (bb_dominates(ep, b2, b1, ++C->L->bb_generation))
+		return cse_one_instruction(C, i1, i2);
+
+	/* No direct dominance - but we could try to find a common ancestor.. */
+	common = trivial_common_parent(b1, b2);
+	if (common) {
+		i1 = cse_one_instruction(C, i2, i1);
+		remove_instruction(&b1->insns, i1, 1);
+		add_instruction_to_end(C, i1, common);
+	}
+
+	return i1;
+}
+
+void dmrC_cleanup_and_cse(struct dmr_C *C, struct entrypoint *ep)
+{
+	int i;
+
+	dmrC_simplify_memops(C, ep);
+repeat:
+	C->L->repeat_phase = 0;
+	clean_up_insns(C, ep);
+	if (C->L->repeat_phase & REPEAT_CFG_CLEANUP)
+		dmrC_kill_unreachable_bbs(C, ep);
+	for (i = 0; i < INSN_HASH_SIZE; i++) {
+		struct instruction_list **list = C->L->insn_hash_table + i;
+		if (*list) {
+			if (dmrC_instruction_list_size(*list) > 1) {
+				struct instruction *insn, *last;
+
+				sort_instruction_list(C, list);
+
+				last = NULL;
+				FOR_EACH_PTR(*list, insn) {
+					if (!insn->bb)
+						continue;
+					if (last) {
+						if (!insn_compare(C, last, insn))
+							insn = try_to_cse(C, ep, last, insn);
+					}
+					last = insn;
+				} END_FOR_EACH_PTR(insn);
+			}
+			ptrlist_remove_all((struct ptr_list **)list);
+		}
+	}
+
+	if (C->L->repeat_phase & REPEAT_SYMBOL_CLEANUP)
+		dmrC_simplify_memops(C, ep);
+
+	if (C->L->repeat_phase & REPEAT_CSE)
+		goto repeat;
+}

dmr_c/src/expand.h

@@ -0,0 +1,34 @@
+#ifndef EXPAND_H
+#define EXPAND_H
+/*
+ * sparse/expand.h
+ *
+ * Copyright (C) 2003 Transmeta Corp.
+ *               2003 Linus Torvalds
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/* Random cost numbers */
+#define SIDE_EFFECTS 10000	/* The expression has side effects */
+#define UNSAFE 100		/* The expression may be "infinitely costly" due to exceptions */
+#define SELECT_COST 20		/* Cut-off for turning a conditional into a select */
+#define BRANCH_COST 10		/* Cost of a conditional branch */
+
+#endif

dmr_c/src/expression.c

@@ -0,0 +1,930 @@
+/*
+ * sparse/expression.c
+ *
+ * Copyright (C) 2003 Transmeta Corp.
+ *               2003-2004 Linus Torvalds
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * This is the expression parsing part of parsing C.
+ */
+ /*
+ * This version is part of the dmr_c project.
+ * Copyright (C) 2017 Dibyendu Majumdar
+ */
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <limits.h>
+
+#include <port.h>
+#include <lib.h>
+#include <allocate.h>
+#include <token.h>
+#include <parse.h>
+#include <symbol.h>
+#include <scope.h>
+#include <expression.h>
+#include <target.h>
+#include <char.h>
+
+static int match_oplist(int op, ...)
+{
+	va_list args;
+	int nextop;
+
+	va_start(args, op);
+	do {
+		nextop = va_arg(args, int);
+	} while (nextop != 0 && nextop != op);
+	va_end(args);
+
+	return nextop != 0;
+}
+
+static struct token *comma_expression(struct dmr_C *C, struct token *, struct expression **);
+
+struct token *dmrC_parens_expression(struct dmr_C *C, struct token *token, struct expression **expr, const char *where)
+{
+	token = dmrC_expect_token(C, token, '(', where);
+	if (dmrC_match_op(token, '{')) {
+		struct expression *e = dmrC_alloc_expression(C, token->pos, EXPR_STATEMENT);
+		struct statement *stmt = dmrC_alloc_statement(C, token->pos, STMT_COMPOUND);
+		*expr = e;
+		e->statement = stmt;
+		dmrC_start_symbol_scope(C);
+		token = dmrC_compound_statement(C, token->next, stmt);
+		dmrC_end_symbol_scope(C);
+		token = dmrC_expect_token(C, token, '}', "at end of statement expression");
+	} else
+		token = dmrC_parse_expression(C, token, expr);
+	return dmrC_expect_token(C, token, ')', where);
+}
+
+/*
+ * Handle __func__, __FUNCTION__ and __PRETTY_FUNCTION__ token
+ * conversion
+ */
+static struct symbol *handle_func(struct dmr_C *C, struct token *token)
+{
+	struct ident *ident = token->ident;
+	struct symbol *decl, *array;
+	struct string *string;
+	int len;
+
+	if (ident != C->S->__func___ident &&
+	    ident != C->S->__FUNCTION___ident &&
+	    ident != C->S->__PRETTY_FUNCTION___ident)
+		return NULL;
+
+	if (!C->current_fn || !C->current_fn->ident)
+		return NULL;
+
+	/* OK, it's one of ours */
+	array = dmrC_alloc_symbol(C->S, token->pos, SYM_ARRAY);
+	array->ctype.base_type = &C->S->char_ctype;
+	array->ctype.alignment = 1;
+	array->endpos = token->pos;
+	decl = dmrC_alloc_symbol(C->S, token->pos, SYM_NODE);
+	decl->ctype.base_type = array;
+	decl->ctype.alignment = 1;
+	decl->ctype.modifiers = MOD_STATIC;
+	decl->endpos = token->pos;
+
+	/* function-scope, but in NS_SYMBOL */
+	dmrC_bind_symbol(C->S, decl, ident, NS_LABEL);
+	decl->ns = NS_SYMBOL;
+
+	len = C->current_fn->ident->len;
+	string = (struct string *)dmrC_allocator_allocate(&C->string_allocator, len + 1);
+	memcpy(string->data, C->current_fn->ident->name, len);
+	string->data[len] = 0;
+	string->length = len + 1;
+
+	decl->initializer = dmrC_alloc_expression(C, token->pos, EXPR_STRING);
+	decl->initializer->string = string;
+	decl->initializer->ctype = decl;
+	decl->array_size = dmrC_alloc_const_expression(C, token->pos, len + 1);
+	array->array_size = decl->array_size;
+	decl->bit_size = array->bit_size = dmrC_bytes_to_bits(C->target, len + 1);
+
+	return decl;
+}
+
+static struct token *parse_type(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	struct symbol *sym;
+	*tree = dmrC_alloc_expression(C, token->pos, EXPR_TYPE);
+	(*tree)->flags = Int_const_expr; /* sic */
+	token = dmrC_typename(C, token, &sym, NULL);
+	if (sym->ident)
+		dmrC_sparse_error(C, token->pos,
+			     "type expression should not include identifier "
+			     "\"%s\"", sym->ident->name);
+	(*tree)->symbol = sym;
+	return token;
+}
+
+static struct token *builtin_types_compatible_p_expr(struct dmr_C *C, struct token *token,
+						     struct expression **tree)
+{
+	struct expression *expr = dmrC_alloc_expression(
+		C, token->pos, EXPR_COMPARE);
+	expr->flags = Int_const_expr;
+	expr->op = SPECIAL_EQUAL;
+	token = token->next;
+	if (!dmrC_match_op(token, '('))
+		return dmrC_expect_token(C, token, '(',
+			      "after __builtin_types_compatible_p");
+	token = token->next;
+	token = parse_type(C, token, &expr->left);
+	if (!dmrC_match_op(token, ','))
+		return dmrC_expect_token(C, token, ',',
+			      "in __builtin_types_compatible_p");
+	token = token->next;
+	token = parse_type(C, token, &expr->right);
+	if (!dmrC_match_op(token, ')'))
+		return dmrC_expect_token(C, token, ')',
+			      "at end of __builtin_types_compatible_p");
+	token = token->next;
+	
+	*tree = expr;
+	return token;
+}
+
+static struct token *builtin_offsetof_expr(struct dmr_C *C, struct token *token,
+					   struct expression **tree)
+{
+	struct expression *expr = NULL;
+	struct expression **p = &expr;
+	struct symbol *sym;
+	int op = '.';
+
+	token = token->next;
+	if (!dmrC_match_op(token, '('))
+		return dmrC_expect_token(C, token, '(', "after __builtin_offset");
+
+	token = token->next;
+	token = dmrC_typename(C, token, &sym, NULL);
+	if (sym->ident)
+		dmrC_sparse_error(C, token->pos,
+			     "type expression should not include identifier "
+			     "\"%s\"", sym->ident->name);
+
+	if (!dmrC_match_op(token, ','))
+		return dmrC_expect_token(C, token, ',', "in __builtin_offset");
+
+	while (1) {
+		struct expression *e;
+		switch (op) {
+		case ')':
+			expr->in = sym;
+			*tree = expr;
+		default:
+			return dmrC_expect_token(C, token, ')', "at end of __builtin_offset");
+		case SPECIAL_DEREFERENCE:
+			e = dmrC_alloc_expression(C, token->pos, EXPR_OFFSETOF);
+			e->flags = Int_const_expr;
+			e->op = '[';
+			*p = e;
+			p = &e->down;
+			/* fall through */
+		case '.':
+			token = token->next;
+			e = dmrC_alloc_expression(C, token->pos, EXPR_OFFSETOF);
+			e->flags = Int_const_expr;
+			e->op = '.';
+			if (dmrC_token_type(token) != TOKEN_IDENT) {
+				dmrC_sparse_error(C, token->pos, "Expected member name");
+				return token;
+			}
+			e->ident = token->ident;
+			token = token->next;
+			break;
+		case '[':
+			token = token->next;
+			e = dmrC_alloc_expression(C, token->pos, EXPR_OFFSETOF);
+			e->flags = Int_const_expr;
+			e->op = '[';
+			token = dmrC_parse_expression(C, token, &e->index);
+			token = dmrC_expect_token(C, token, ']',
+					"at end of array dereference");
+			if (!e->index)
+				return token;
+		}
+		*p = e;
+		p = &e->down;
+		op = dmrC_token_type(token) == TOKEN_SPECIAL ? token->special : 0;
+	}
+}
+
+#ifndef ULLONG_MAX
+#define ULLONG_MAX (~0ULL)
+#endif
+
+static unsigned long long parse_num(const char *nptr, char **end)
+{
+	if (nptr[0] == '0' && tolower((unsigned char)nptr[1]) == 'b')
+		return strtoull(&nptr[2], end, 2);
+	return strtoull(nptr, end, 0);
+}
+
+static void get_number_value(struct dmr_C *C, struct expression *expr, struct token *token)
+{
+	const char *str = token->number;
+	unsigned long long value;
+	char *end;
+	int size = 0, want_unsigned = 0;
+	int overflow = 0, do_warn = 0;
+	int try_unsigned = 1;
+	int bits;
+
+	errno = 0;
+	value = parse_num(str, &end);
+	if (end == str)
+		goto Float;
+	if (value == ULLONG_MAX && errno == ERANGE)
+		overflow = 1;
+	while (1) {
+		char c = *end++;
+		if (!c) {
+			break;
+		} else if (c == 'u' || c == 'U') {
+			if (want_unsigned)
+				goto Enoint;
+			want_unsigned = 1;
+		} else if (c == 'l' || c == 'L') {
+			if (size)
+				goto Enoint;
+			size = 1;
+			if (*end == c) {
+				size = 2;
+				end++;
+			}
+		} else
+			goto Float;
+	}
+	if (overflow)
+		goto Eoverflow;
+	/* OK, it's a valid integer */
+	/* decimals can be unsigned only if directly specified as such */
+	if (str[0] != '0' && !want_unsigned)
+		try_unsigned = 0;
+	if (!size) {
+		bits = C->target->bits_in_int - 1;
+		if (!(value & (~1ULL << bits))) {
+			if (!(value & (1ULL << bits))) {
+				goto got_it;
+			} else if (try_unsigned) {
+				want_unsigned = 1;
+				goto got_it;
+			}
+		}
+		size = 1;
+		do_warn = 1;
+	}
+	if (size < 2) {
+		bits = C->target->bits_in_long - 1;
+		if (!(value & (~1ULL << bits))) {
+			if (!(value & (1ULL << bits))) {
+				goto got_it;
+			} else if (try_unsigned) {
+				want_unsigned = 1;
+				goto got_it;
+			}
+			do_warn |= 2;
+		}
+		size = 2;
+		do_warn |= 1;
+	}
+	bits = C->target->bits_in_longlong - 1;
+	if (value & (~1ULL << bits))
+		goto Eoverflow;
+	if (!(value & (1ULL << bits)))
+		goto got_it;
+	if (!try_unsigned)
+		dmrC_warning(C, expr->pos, "decimal constant %s is too big for long long",
+			dmrC_show_token(C, token));
+	want_unsigned = 1;
+got_it:
+	if (do_warn)
+		dmrC_warning(C, expr->pos, "constant %s is so big it is%s%s%s",
+			dmrC_show_token(C, token),
+			want_unsigned ? " unsigned":"",
+			size > 0 ? " long":"",
+			size > 1 ? " long":"");
+	if (do_warn & 2)
+		dmrC_warning(C, expr->pos,
+			"decimal constant %s is between LONG_MAX and ULONG_MAX."
+			" For C99 that means long long, C90 compilers are very "
+			"likely to produce unsigned long (and a warning) here",
+			dmrC_show_token(C, token));
+        expr->type = EXPR_VALUE;
+	expr->flags = Int_const_expr;
+        expr->ctype = dmrC_ctype_integer(C, size, want_unsigned);
+        expr->value = value;
+	return;
+Eoverflow:
+	dmrC_error_die(C, expr->pos, "constant %s is too big even for unsigned long long",
+			dmrC_show_token(C, token));
+	return;
+Float:
+	expr->fvalue = dmrC_string_to_ld(str, &end);
+	if (str == end)
+		goto Enoint;
+
+	if (*end && end[1])
+		goto Enoint;
+
+	if (*end == 'f' || *end == 'F')
+		expr->ctype = &C->S->float_ctype;
+	else if (*end == 'l' || *end == 'L')
+		expr->ctype = &C->S->ldouble_ctype;
+	else if (!*end)
+		expr->ctype = &C->S->double_ctype;
+	else
+		goto Enoint;
+
+	expr->flags = Float_literal;
+	expr->type = EXPR_FVALUE;
+	return;
+
+Enoint:
+	dmrC_error_die(C, expr->pos, "constant %s is not a valid number", dmrC_show_token(C, token));
+}
+
+struct token *dmrC_primary_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	struct expression *expr = NULL;
+
+	switch (dmrC_token_type(token)) {
+	case TOKEN_CHAR:
+	case TOKEN_CHAR_EMBEDDED_0:
+	case TOKEN_CHAR_EMBEDDED_1:
+	case TOKEN_CHAR_EMBEDDED_2:
+	case TOKEN_CHAR_EMBEDDED_3:
+	case TOKEN_WIDE_CHAR:
+	case TOKEN_WIDE_CHAR_EMBEDDED_0:
+	case TOKEN_WIDE_CHAR_EMBEDDED_1:
+	case TOKEN_WIDE_CHAR_EMBEDDED_2:
+	case TOKEN_WIDE_CHAR_EMBEDDED_3:
+		expr = dmrC_alloc_expression(C, token->pos, EXPR_VALUE);   
+		expr->flags = Int_const_expr;
+		expr->ctype = dmrC_token_type(token) < TOKEN_WIDE_CHAR ? &C->S->int_ctype : &C->S->long_ctype;
+		dmrC_get_char_constant(C, token, &expr->value);
+		token = token->next;
+		break;
+
+	case TOKEN_NUMBER:
+		expr = dmrC_alloc_expression(C, token->pos, EXPR_VALUE);
+		get_number_value(C, expr, token); /* will see if it's an integer */
+		token = token->next;
+		break;
+
+	case TOKEN_ZERO_IDENT: {
+		expr = dmrC_alloc_expression(C, token->pos, EXPR_SYMBOL);
+		expr->flags = Int_const_expr;
+		expr->ctype = &C->S->int_ctype;
+		expr->symbol = &C->S->zero_int;
+		expr->symbol_name = token->ident;
+		token = token->next;
+		break;
+	}
+
+	case TOKEN_IDENT: {
+		struct symbol *sym = dmrC_lookup_symbol(token->ident, NS_SYMBOL | NS_TYPEDEF);
+		struct token *next = token->next;
+
+		if (!sym) {
+			sym = handle_func(C, token);
+			if (token->ident == C->S->__builtin_types_compatible_p_ident) {
+				token = builtin_types_compatible_p_expr(C, token, &expr);
+				break;
+			}
+			if (token->ident == C->S->__builtin_offsetof_ident) {
+				token = builtin_offsetof_expr(C, token, &expr);
+				break;
+			}
+		} else if (sym->enum_member) {
+			expr = dmrC_alloc_expression(C, token->pos, EXPR_VALUE);
+			*expr = *sym->initializer;
+			/* we want the right position reported, thus the copy */
+			expr->pos = token->pos;
+			expr->flags = Int_const_expr;
+			token = next;
+			break;
+		}
+
+		expr = dmrC_alloc_expression(C, token->pos, EXPR_SYMBOL);
+
+		/*
+		 * We support types as real first-class citizens, with type
+		 * comparisons etc:
+		 *
+		 *	if (typeof(a) == int) ..
+		 */
+		if (sym && sym->ns == NS_TYPEDEF) {
+			dmrC_sparse_error(C, token->pos, "typename in expression");
+			sym = NULL;
+		}
+		expr->symbol_name = token->ident;
+		expr->symbol = sym;
+		token = next;
+		break;
+	}
+
+	case TOKEN_STRING:
+	case TOKEN_WIDE_STRING:
+		expr = dmrC_alloc_expression(C, token->pos, EXPR_STRING);
+		token = dmrC_get_string_constant(C, token, expr);
+		break;
+
+	case TOKEN_SPECIAL:
+		if (token->special == '(') {
+			expr = dmrC_alloc_expression(C, token->pos, EXPR_PREOP);
+			expr->op = '(';
+			token = dmrC_parens_expression(C, token, &expr->unop, "in expression");
+			if (expr->unop)
+				expr->flags = expr->unop->flags;
+			break;
+		}
+		if (token->special == '[' && dmrC_lookup_type(token->next)) {
+			expr = dmrC_alloc_expression(C, token->pos, EXPR_TYPE);
+			expr->flags = Int_const_expr; /* sic */
+			token = dmrC_typename(C, token->next, &expr->symbol, NULL);
+			token = dmrC_expect_token(C, token, ']', "in type expression");
+			break;
+		}
+			
+	default:
+		;
+	}
+	*tree = expr;
+	return token;
+}
+
+static struct token *expression_list(struct dmr_C *C, struct token *token, struct expression_list **list)
+{
+	while (!dmrC_match_op(token, ')')) {
+		struct expression *expr = NULL;
+		token = dmrC_assignment_expression(C, token, &expr);
+		if (!expr)
+			break;
+		dmrC_add_expression(C, list, expr);
+		if (!dmrC_match_op(token, ','))
+			break;
+		token = token->next;
+	}
+	return token;
+}
+
+/*
+ * extend to deal with the ambiguous C grammar for parsing
+ * a cast expressions followed by an initializer.
+ */
+static struct token *postfix_expression(struct dmr_C *C, struct token *token, struct expression **tree, struct expression *cast_init_expr)
+{
+	struct expression *expr = cast_init_expr;
+
+	if (!expr)
+		token = dmrC_primary_expression(C, token, &expr);
+
+	while (expr && dmrC_token_type(token) == TOKEN_SPECIAL) {
+		switch (token->special) {
+		case '[': {			/* Array dereference */
+			struct expression *deref = dmrC_alloc_expression(C, token->pos, EXPR_PREOP);
+			struct expression *add = dmrC_alloc_expression(C, token->pos, EXPR_BINOP);
+
+			deref->op = '*';
+			deref->unop = add;
+
+			add->op = '+';
+			add->left = expr;
+			token = dmrC_parse_expression(C, token->next, &add->right);
+			token = dmrC_expect_token(C, token, ']', "at end of array dereference");
+			expr = deref;
+			continue;
+		}
+		case SPECIAL_INCREMENT:		/* Post-increment */
+		case SPECIAL_DECREMENT:	{	/* Post-decrement */
+			struct expression *post = dmrC_alloc_expression(C, token->pos, EXPR_POSTOP);
+			post->op = token->special;
+			post->unop = expr;
+			expr = post;
+			token = token->next;
+			continue;
+		}
+		case SPECIAL_DEREFERENCE: {	/* Structure pointer member dereference */
+			/* "x->y" is just shorthand for "(*x).y" */
+			struct expression *inner = dmrC_alloc_expression(C, token->pos, EXPR_PREOP);
+			inner->op = '*';
+			inner->unop = expr;
+			expr = inner;
+		}
+		/* Fall through!! */
+		case '.': {			/* Structure member dereference */
+			struct expression *deref = dmrC_alloc_expression(C, token->pos, EXPR_DEREF);
+			deref->op = '.';
+			deref->deref = expr;
+			token = token->next;
+			if (dmrC_token_type(token) != TOKEN_IDENT) {
+				dmrC_sparse_error(C, token->pos, "Expected member name");
+				break;
+			}
+			deref->member = token->ident;
+			token = token->next;
+			expr = deref;
+			continue;
+		}
+
+		case '(': {			/* Function call */
+			struct expression *call = dmrC_alloc_expression(C, token->pos, EXPR_CALL);
+			call->op = '(';
+			call->fn = expr;
+			token = expression_list(C, token->next, &call->args);
+			token = dmrC_expect_token(C, token, ')', "in function call");
+			expr = call;
+			continue;
+		}
+
+		default:
+			break;
+		}
+		break;
+	}
+	*tree = expr;
+	return token;
+}
+
+static struct token *cast_expression(struct dmr_C *C, struct token *token, struct expression **tree);
+static struct token *unary_expression(struct dmr_C *C, struct token *token, struct expression **tree);
+
+static struct token *type_info_expression(struct dmr_C *C, struct token *token,
+	struct expression **tree, int type)
+{
+	struct expression *expr = dmrC_alloc_expression(C, token->pos, type);
+	struct token *p;
+
+	*tree = expr;
+	expr->flags = Int_const_expr; /* XXX: VLA support will need that changed */
+	token = token->next;
+	if (!dmrC_match_op(token, '(') || !dmrC_lookup_type(token->next))
+		return unary_expression(C, token, &expr->cast_expression);
+	p = token;
+	token = dmrC_typename(C, token->next, &expr->cast_type, NULL);
+
+	if (!dmrC_match_op(token, ')')) {
+		static const char * error[] = {
+			[EXPR_SIZEOF] = "at end of sizeof",
+			[EXPR_ALIGNOF] = "at end of __alignof__",
+			[EXPR_PTRSIZEOF] = "at end of __sizeof_ptr__"
+		};
+		return dmrC_expect_token(C, token, ')', error[type]);
+	}
+
+	token = token->next;
+	/*
+	 * C99 ambiguity: the typename might have been the beginning
+	 * of a typed initializer expression..
+	 */
+	if (dmrC_match_op(token, '{')) {
+		struct expression *cast = dmrC_alloc_expression(C, p->pos, EXPR_CAST);
+		cast->cast_type = expr->cast_type;
+		expr->cast_type = NULL;
+		expr->cast_expression = cast;
+		token = dmrC_initializer(C, &cast->cast_expression, token);
+		token = postfix_expression(C, token, &expr->cast_expression, cast);
+	}
+	return token;
+}
+
+static struct token *unary_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	if (dmrC_token_type(token) == TOKEN_IDENT) {
+		struct ident *ident = token->ident;
+		if (ident->reserved) {
+			const struct {
+				struct ident *id;
+				int type;
+			} type_information[] = {
+				{ C->S->sizeof_ident, EXPR_SIZEOF },
+				{ C->S->__alignof___ident, EXPR_ALIGNOF },
+				{ C->S->__alignof_ident, EXPR_ALIGNOF },
+				{ C->S->_Alignof_ident, EXPR_ALIGNOF },
+				{ C->S->__sizeof_ptr___ident, EXPR_PTRSIZEOF },
+			};
+			int i;
+			for (i = 0; i < (int)ARRAY_SIZE(type_information); i++) {
+				if (ident == type_information[i].id)
+					return type_info_expression(C, token, tree, type_information[i].type);
+			}
+		}
+	}
+
+	if (dmrC_token_type(token) == TOKEN_SPECIAL) {
+		if (match_oplist(token->special,
+		    SPECIAL_INCREMENT, SPECIAL_DECREMENT,
+		    '&', '*', 0)) {
+		    	struct expression *unop;
+			struct expression *unary;
+			struct token *next;
+
+			next = cast_expression(C, token->next, &unop);
+			if (!unop) {
+				dmrC_sparse_error(C, token->pos, "Syntax error in unary expression");
+				*tree = NULL;
+				return next;
+			}
+			unary = dmrC_alloc_expression(C, token->pos, EXPR_PREOP);
+			unary->op = token->special;
+			unary->unop = unop;
+			*tree = unary;
+			return next;
+		}
+		/* possibly constant ones */
+		if (match_oplist(token->special, '+', '-', '~', '!', 0)) {
+		    	struct expression *unop;
+			struct expression *unary;
+			struct token *next;
+
+			next = cast_expression(C, token->next, &unop);
+			if (!unop) {
+				dmrC_sparse_error(C, token->pos, "Syntax error in unary expression");
+				*tree = NULL;
+				return next;
+			}
+			unary = dmrC_alloc_expression(C, token->pos, EXPR_PREOP);
+			unary->op = token->special;
+			unary->unop = unop;
+			unary->flags = unop->flags & Int_const_expr;
+			*tree = unary;
+			return next;
+		}
+		/* Gcc extension: &&label gives the address of a label */
+		if (dmrC_match_op(token, SPECIAL_LOGICAL_AND) &&
+		    dmrC_token_type(token->next) == TOKEN_IDENT) {
+			struct expression *label = dmrC_alloc_expression(C, token->pos, EXPR_LABEL);
+			struct symbol *sym = dmrC_label_symbol(C, token->next);
+			if (!(sym->ctype.modifiers & MOD_ADDRESSABLE)) {
+				sym->ctype.modifiers |= MOD_ADDRESSABLE;
+				dmrC_add_symbol(C, &C->P->function_computed_target_list, sym);
+			}
+			label->label_symbol = sym;
+			*tree = label;
+			return token->next->next;
+		}
+						
+	}
+			
+	return postfix_expression(C, token, tree, NULL);
+}
+
+/*
+ * Ambiguity: a '(' can be either a cast-expression or
+ * a primary-expression depending on whether it is followed
+ * by a type or not. 
+ *
+ * additional ambiguity: a "cast expression" followed by
+ * an initializer is really a postfix-expression.
+ */
+static struct token *cast_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	if (dmrC_match_op(token, '(')) {
+		struct token *next = token->next;
+		if (dmrC_lookup_type(next)) {
+			struct expression *cast = dmrC_alloc_expression(C, next->pos, EXPR_CAST);
+			struct expression *v;
+			struct symbol *sym;
+			int is_force;
+
+			token = dmrC_typename(C, next, &sym, &is_force);
+			cast->cast_type = sym;
+			token = dmrC_expect_token(C, token, ')', "at end of cast operator");
+			if (dmrC_match_op(token, '{')) {
+				if (is_force)
+					dmrC_warning(C, sym->pos,
+						"[force] in compound literal");
+				token = dmrC_initializer(C, &cast->cast_expression, token);
+				return postfix_expression(C, token, tree, cast);
+			}
+			*tree = cast;
+			if (is_force)
+				cast->type = EXPR_FORCE_CAST;
+			token = cast_expression(C, token, &v);
+			if (!v)
+				return token;
+			cast->cast_expression = v;
+			if (v->flags & Int_const_expr)
+				cast->flags = Int_const_expr;
+			else if (v->flags & Float_literal) /* and _not_ int */
+				cast->flags = Int_const_expr | Float_literal;
+			return token;
+		}
+	}
+	return unary_expression(C, token, tree);
+}
+
+/*
+ * Generic left-to-right binop parsing
+ *
+ * This _really_ needs to be inlined, because that makes the inner
+ * function call statically deterministic rather than a totally
+ * unpredictable indirect call. But gcc-3 is so "clever" that it
+ * doesn't do so by default even when you tell it to inline it.
+ *
+ * Making it a macro avoids the inlining problem, and also means
+ * that we can pass in the op-comparison as an expression rather
+ * than create a data structure for it.
+ */
+
+#define LR_BINOP_EXPRESSION(C, __token, tree, type, inner, compare)	\
+	struct expression *left = NULL;					\
+	struct token * next = inner(C, __token, &left);			\
+									\
+	if (left) {							\
+		while (dmrC_token_type(next) == TOKEN_SPECIAL) {		\
+			struct expression *top, *right = NULL;		\
+			int op = next->special;				\
+									\
+			if (!(compare))					\
+				goto out;				\
+			top = dmrC_alloc_expression(C, next->pos, type);	\
+			next = inner(C, next->next, &right);		\
+			if (!right) {					\
+				dmrC_sparse_error(C, next->pos, "No right hand side of '%s'-expression", dmrC_show_special(C, op));	\
+				break;					\
+			}						\
+			top->flags = left->flags & right->flags		\
+						& Int_const_expr;	\
+			top->op = op;					\
+			top->left = left;				\
+			top->right = right;				\
+			left = top;					\
+		}							\
+	}								\
+out:									\
+	*tree = left;							\
+	return next;							\
+
+static struct token *multiplicative_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C, 
+		token, tree, EXPR_BINOP, cast_expression,
+		(op == '*') || (op == '/') || (op == '%')
+	);
+}
+
+static struct token *additive_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C, 
+		token, tree, EXPR_BINOP, multiplicative_expression,
+		(op == '+') || (op == '-')
+	);
+}
+
+static struct token *shift_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C, 
+		token, tree, EXPR_BINOP, additive_expression,
+		(op == SPECIAL_LEFTSHIFT) || (op == SPECIAL_RIGHTSHIFT)
+	);
+}
+
+static struct token *relational_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C, 
+		token, tree, EXPR_COMPARE, shift_expression,
+		(op == '<') || (op == '>') ||
+		(op == SPECIAL_LTE) || (op == SPECIAL_GTE)
+	);
+}
+
+static struct token *equality_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C, 
+		token, tree, EXPR_COMPARE, relational_expression,
+		(op == SPECIAL_EQUAL) || (op == SPECIAL_NOTEQUAL)
+	);
+}
+
+static struct token *bitwise_and_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C,
+		token, tree, EXPR_BINOP, equality_expression,
+		(op == '&')
+	);
+}
+
+static struct token *bitwise_xor_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C,
+		token, tree, EXPR_BINOP, bitwise_and_expression,
+		(op == '^')
+	);
+}
+
+static struct token *bitwise_or_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C,
+		token, tree, EXPR_BINOP, bitwise_xor_expression,
+		(op == '|')
+	);
+}
+
+static struct token *logical_and_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C,
+		token, tree, EXPR_LOGICAL, bitwise_or_expression,
+		(op == SPECIAL_LOGICAL_AND)
+	);
+}
+
+static struct token *logical_or_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C,
+		token, tree, EXPR_LOGICAL, logical_and_expression,
+		(op == SPECIAL_LOGICAL_OR)
+	);
+}
+
+struct token *dmrC_conditional_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	token = logical_or_expression(C, token, tree);
+	if (*tree && dmrC_match_op(token, '?')) {
+		struct expression *expr = dmrC_alloc_expression(C, token->pos, EXPR_CONDITIONAL);
+		expr->op = token->special;
+		expr->left = *tree;
+		*tree = expr;
+		token = dmrC_parse_expression(C, token->next, &expr->cond_true);
+		token = dmrC_expect_token(C, token, ':', "in conditional expression");
+		token = dmrC_conditional_expression(C, token, &expr->cond_false);
+		if (expr->left && expr->cond_false) {
+			int is_const = expr->left->flags &
+					expr->cond_false->flags &
+					Int_const_expr;
+			if (expr->cond_true)
+				is_const &= expr->cond_true->flags;
+			expr->flags = is_const;
+		}
+	}
+	return token;
+}
+
+struct token *dmrC_assignment_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	token = dmrC_conditional_expression(C, token, tree);
+	if (*tree && dmrC_token_type(token) == TOKEN_SPECIAL) {
+		static const int assignments[] = {
+			'=',
+			SPECIAL_ADD_ASSIGN, SPECIAL_SUB_ASSIGN,
+			SPECIAL_MUL_ASSIGN, SPECIAL_DIV_ASSIGN,
+			SPECIAL_MOD_ASSIGN, SPECIAL_SHL_ASSIGN,
+			SPECIAL_SHR_ASSIGN, SPECIAL_AND_ASSIGN,
+			SPECIAL_OR_ASSIGN,  SPECIAL_XOR_ASSIGN };
+		int i, op = token->special;
+		for (i = 0; i < (int)ARRAY_SIZE(assignments); i++)
+			if (assignments[i] == op) {
+				struct expression * expr = dmrC_alloc_expression(C, token->pos, EXPR_ASSIGNMENT);
+				expr->left = *tree;
+				expr->op = op;
+				*tree = expr;
+				return dmrC_assignment_expression(C, token->next, &expr->right);
+			}
+	}
+	return token;
+}
+
+static struct token *comma_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	LR_BINOP_EXPRESSION(C,
+		token, tree, EXPR_COMMA, dmrC_assignment_expression,
+		(op == ',')
+	);
+}
+
+struct token *dmrC_parse_expression(struct dmr_C *C, struct token *token, struct expression **tree)
+{
+	return comma_expression(C, token,tree);
+}
+
+

dmr_c/src/expression.h

@@ -0,0 +1,265 @@
+#ifndef DMR_C_EXPRESSION_H
+#define DMR_C_EXPRESSION_H
+/*
+* sparse/expression.h
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*
+* Declarations and helper functions for expression parsing.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+
+#include <allocate.h>
+#include <lib.h>
+#include <symbol.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum expression_type {
+	EXPR_VALUE = 1,
+	EXPR_STRING,
+	EXPR_SYMBOL,
+	EXPR_TYPE,
+	EXPR_BINOP,
+	EXPR_ASSIGNMENT,
+	EXPR_LOGICAL,
+	EXPR_DEREF,
+	EXPR_PREOP,
+	EXPR_POSTOP,
+	EXPR_CAST,
+	EXPR_FORCE_CAST,
+	EXPR_IMPLIED_CAST,
+	EXPR_SIZEOF,
+	EXPR_ALIGNOF,
+	EXPR_PTRSIZEOF,
+	EXPR_CONDITIONAL,
+	EXPR_SELECT,		// a "safe" conditional expression
+	EXPR_STATEMENT,
+	EXPR_CALL,
+	EXPR_COMMA,
+	EXPR_COMPARE,
+	EXPR_LABEL,
+	EXPR_INITIALIZER,	// initializer list
+	EXPR_IDENTIFIER,	// identifier in initializer
+	EXPR_INDEX,		// index in initializer
+	EXPR_POS,		// position in initializer
+	EXPR_FVALUE,
+	EXPR_SLICE,
+	EXPR_OFFSETOF,
+};
+
+enum {
+	Int_const_expr = 1,
+	Float_literal = 2,
+}; /* for expr->flags */
+
+enum {
+	Taint_comma = 1,
+}; /* for expr->taint */
+
+DECLARE_PTR_LIST(expression_list, struct expression);
+
+struct expression {
+	enum expression_type type:8;
+	unsigned flags:8;
+	int op;
+	struct position pos;
+	struct symbol *ctype;
+	union {
+		// EXPR_VALUE
+		struct {
+			unsigned long long value;
+			unsigned taint;
+		};
+
+		// EXPR_FVALUE
+		long double fvalue;
+
+		// EXPR_STRING
+		struct {
+			int wide;
+			struct string *string;
+		};
+
+		// EXPR_UNOP, EXPR_PREOP and EXPR_POSTOP
+		struct /* unop */ {
+			struct expression *unop;
+			unsigned long op_value;
+		};
+
+		// EXPR_SYMBOL, EXPR_TYPE
+		struct /* symbol_arg */ {
+			struct symbol *symbol;
+			struct ident *symbol_name;
+		};
+
+		// EXPR_STATEMENT
+		struct statement *statement;
+
+		// EXPR_BINOP, EXPR_COMMA, EXPR_COMPARE, EXPR_LOGICAL and EXPR_ASSIGNMENT
+		struct /* binop_arg */ {
+			struct expression *left, *right;
+		};
+		// EXPR_DEREF
+		struct /* deref_arg */ {
+			struct expression *deref;
+			struct ident *member;
+		};
+		// EXPR_SLICE
+		struct /* slice */ {
+			struct expression *base;
+			unsigned r_bitpos, r_nrbits;
+		};
+		// EXPR_CAST and EXPR_SIZEOF
+		struct /* cast_arg */ {
+			struct symbol *cast_type;
+			struct expression *cast_expression;
+		};
+		// EXPR_CONDITIONAL
+		// EXPR_SELECT
+		struct /* conditional_expr */ {
+			struct expression *conditional, *cond_true, *cond_false;
+		};
+		// EXPR_CALL
+		struct /* call_expr */ {
+			struct expression *fn;
+			struct expression_list *args;
+		};
+		// EXPR_LABEL
+		struct /* label_expr */ {
+			struct symbol *label_symbol;
+		};
+		// EXPR_INITIALIZER
+		struct expression_list *expr_list;
+		// EXPR_IDENTIFIER
+		struct /* ident_expr */ {
+			int offset;
+			struct ident *expr_ident;
+			struct symbol *field;
+			struct expression *ident_expression;
+		};
+		// EXPR_INDEX
+		struct /* index_expr */ {
+			unsigned int idx_from, idx_to;
+			struct expression *idx_expression;
+		};
+		// EXPR_POS
+		struct /* initpos_expr */ {
+			unsigned int init_offset, init_nr;
+			struct expression *init_expr;
+		};
+		// EXPR_OFFSETOF
+		struct {
+			struct symbol *in;
+			struct expression *down;
+			union {
+				struct ident *ident;
+				struct expression *index;
+			};
+		};
+	};
+};
+
+long long dmrC_get_expression_value_silent(struct dmr_C *C, struct expression *expr);
+extern struct symbol *dmrC_evaluate_expression(struct dmr_C *C, struct expression *);
+long long dmrC_get_expression_value(struct dmr_C *C, struct expression *);
+
+/* Constant expression values */
+int dmrC_is_zero_constant(struct dmr_C *C, struct expression *);
+int dmrC_expr_truth_value(struct dmr_C *C, struct expression *expr);
+long long dmrC_const_expression_value(struct dmr_C *C, struct expression *);
+
+/* Expression parsing */
+struct token *dmrC_conditional_expression(struct dmr_C *C, struct token *token,
+				     struct expression **tree);
+struct token *dmrC_primary_expression(struct dmr_C *C, struct token *token, struct expression **tree);
+struct token *dmrC_parens_expression(struct dmr_C *C, struct token *token, struct expression **expr,
+				const char *where);
+struct token *dmrC_assignment_expression(struct dmr_C *C, struct token *token,
+				    struct expression **tree);
+
+extern void dmrC_evaluate_symbol_list(struct dmr_C *C, struct symbol_list *list);
+extern struct symbol *dmrC_evaluate_statement(struct dmr_C *C, struct statement *stmt);
+
+extern int dmrC_expand_symbol(struct dmr_C *C, struct symbol *);
+
+static inline struct expression *dmrC_alloc_expression(struct dmr_C *C, struct position pos, int type)
+{
+	struct expression *expr = (struct expression *)dmrC_allocator_allocate(&C->expression_allocator, 0);
+	expr->type = (enum expression_type)type;
+	expr->pos = pos;
+	return expr;
+}
+
+static inline struct expression *dmrC_alloc_const_expression(struct dmr_C *C, struct position pos,
+							int value)
+{
+	struct expression *expr = (struct expression *)dmrC_allocator_allocate(&C->expression_allocator, 0);
+	expr->type = EXPR_VALUE;
+	expr->pos = pos;
+	expr->value = value;
+	expr->ctype = &C->S->int_ctype;
+	return expr;
+}
+
+/* Type name parsing */
+struct token *dmrC_typename(struct dmr_C *C, struct token *, struct symbol **, int *);
+
+static inline int dmrC_lookup_type(struct token *token)
+{
+	if (token->pos.type == TOKEN_IDENT) {
+		struct symbol *sym = dmrC_lookup_symbol(
+		    token->ident,
+		    (enum namespace_type)(NS_SYMBOL | NS_TYPEDEF));
+		return sym && (sym->ns & NS_TYPEDEF);
+	}
+	return 0;
+}
+
+/* Statement parsing */
+struct statement *dmrC_alloc_statement(struct dmr_C *C, struct position pos, int type);
+struct token *dmrC_initializer(struct dmr_C *C, struct expression **tree, struct token *token);
+struct token *dmrC_compound_statement(struct dmr_C *C, struct token *, struct statement *);
+
+/* The preprocessor calls this 'dmrC_constant_expression()' */
+#define dmrC_constant_expression(C, token, tree) dmrC_conditional_expression(C, token, tree)
+
+/* Cast folding of constant values.. */
+void dmrC_cast_value(struct dmr_C *C, struct expression *expr, struct symbol *newtype,
+		struct expression *old, struct symbol *oldtype);
+
+static inline struct expression *dmrC_first_expression(struct expression_list *head)
+{
+	return (struct expression *) ptrlist_first((struct ptr_list *)head);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

dmr_c/src/flow.h

@@ -0,0 +1,46 @@
+#ifndef DMR_C_FLOW_H
+#define DMR_C_FLOW_H
+
+/*
+* Flow - walk the linearized flowgraph, simplifying it as we
+* go along.
+*
+* Copyright (C) 2004 Linus Torvalds
+*/
+
+#include <lib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct entrypoint;
+struct instruction;
+
+extern int dmrC_simplify_flow(struct dmr_C *C, struct entrypoint *ep);
+
+extern void dmrC_simplify_symbol_usage(struct dmr_C *C, struct entrypoint *ep);
+extern void dmrC_simplify_memops(struct dmr_C *C, struct entrypoint *ep);
+extern void dmrC_pack_basic_blocks(struct dmr_C *C, struct entrypoint *ep);
+
+extern void dmrC_cleanup_and_cse(struct dmr_C *C, struct entrypoint *ep);
+extern int dmrC_simplify_instruction(struct dmr_C *C, struct instruction *);
+
+extern void dmrC_kill_use(struct dmr_C *C, pseudo_t *usep);
+extern void dmrC_remove_use(struct dmr_C *C, pseudo_t *);
+
+void dmrC_check_access(struct dmr_C *C, struct instruction *insn);
+void dmrC_convert_load_instruction(struct dmr_C *C, struct instruction *, pseudo_t);
+void dmrC_rewrite_load_instruction(struct dmr_C *C, struct instruction *, struct pseudo_list *);
+int dmrC_dominates(struct dmr_C *C, pseudo_t pseudo, struct instruction *insn, struct instruction *dom, int local);
+
+extern void dmrC_vrfy_flow(struct entrypoint *ep);
+extern int dmrC_pseudo_in_list(struct pseudo_list *list, pseudo_t pseudo);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/gcc-attr-list.h

@@ -0,0 +1,207 @@
+GCC_ATTR(BELOW100)
+GCC_ATTR(OS_Task)
+GCC_ATTR(OS_main)
+GCC_ATTR(OS_task)
+GCC_ATTR(abi_tag)
+GCC_ATTR(absdata)
+GCC_ATTR(address)
+GCC_ATTR(alias)
+GCC_ATTR(aligned)
+GCC_ATTR(alloc_align)
+GCC_ATTR(alloc_size)
+GCC_ATTR(altivec)
+GCC_ATTR(always_inline)
+GCC_ATTR(artificial)
+GCC_ATTR(assume_aligned)
+GCC_ATTR(bank_switch)
+GCC_ATTR(based)
+GCC_ATTR(below100)
+GCC_ATTR(bnd_instrument)
+GCC_ATTR(bnd_legacy)
+GCC_ATTR(bnd_variable_size)
+GCC_ATTR(break_handler)
+GCC_ATTR(brk_interrupt)
+GCC_ATTR(callee_pop_aggregate_return)
+GCC_ATTR(cb)
+GCC_ATTR(cdecl)
+GCC_ATTR(cleanup)
+GCC_ATTR(cmse_nonsecure_call)
+GCC_ATTR(cmse_nonsecure_entry)
+GCC_ATTR(cold)
+GCC_ATTR(common)
+GCC_ATTR(common_object)
+GCC_ATTR(const)
+GCC_ATTR(constructor)
+GCC_ATTR(critical)
+GCC_ATTR(default)
+GCC_ATTR(deprecated)
+GCC_ATTR(designated_init)
+GCC_ATTR(destructor)
+GCC_ATTR(disinterrupt)
+GCC_ATTR(dllexport)
+GCC_ATTR(dllimport)
+GCC_ATTR(eightbit_data)
+GCC_ATTR(either)
+GCC_ATTR(error)
+GCC_ATTR(exception)
+GCC_ATTR(exception_handler)
+GCC_ATTR(externally_visible)
+GCC_ATTR(fallthrough)
+GCC_ATTR(far)
+GCC_ATTR(fast_interrupt)
+GCC_ATTR(fastcall)
+GCC_ATTR(flatten)
+GCC_ATTR(force_align_arg_pointer)
+GCC_ATTR(format)
+GCC_ATTR(format_arg)
+GCC_ATTR(forwarder_section)
+GCC_ATTR(function_vector)
+GCC_ATTR(gcc_struct)
+GCC_ATTR(gnu_inline)
+GCC_ATTR(hidden)
+GCC_ATTR(hot)
+GCC_ATTR(hotpatch)
+GCC_ATTR(ifunc)
+GCC_ATTR(init_priority)
+GCC_ATTR(interfacearm)
+GCC_ATTR(internal)
+GCC_ATTR(interrupt)
+GCC_ATTR(interrupt_handler)
+GCC_ATTR(interrupt_thread)
+GCC_ATTR(io)
+GCC_ATTR(io_low)
+GCC_ATTR(isr)
+GCC_ATTR(keep_interrupts_masked)
+GCC_ATTR(kernel)
+GCC_ATTR(kspisusp)
+GCC_ATTR(l1_data)
+GCC_ATTR(l1_data_A)
+GCC_ATTR(l1_data_B)
+GCC_ATTR(l1_text)
+GCC_ATTR(l2)
+GCC_ATTR(leaf)
+GCC_ATTR(long_call)
+GCC_ATTR(longcall)
+GCC_ATTR(lower)
+GCC_ATTR(malloc)
+GCC_ATTR(may_alias)
+GCC_ATTR(maybe_unused)
+GCC_ATTR(medium_call)
+GCC_ATTR(micromips)
+GCC_ATTR(mips16)
+GCC_ATTR(mode)
+GCC_ATTR(model)
+GCC_ATTR(monitor)
+GCC_ATTR(ms_abi)
+GCC_ATTR(ms_hook_prologue)
+GCC_ATTR(ms_struct)
+GCC_ATTR(naked)
+GCC_ATTR(near)
+GCC_ATTR(nested)
+GCC_ATTR(nested_ready)
+GCC_ATTR(nesting)
+GCC_ATTR(nmi)
+GCC_ATTR(nmi_handler)
+GCC_ATTR(no_address_safety_analysis)
+GCC_ATTR(no_caller_saved_registers)
+GCC_ATTR(no_gccisr)
+GCC_ATTR(no_icf)
+GCC_ATTR(no_instrument_function)
+GCC_ATTR(no_profile_instrument_function)
+GCC_ATTR(no_reorder)
+GCC_ATTR(no_sanitize)
+GCC_ATTR(no_sanitize_address)
+GCC_ATTR(no_sanitize_thread)
+GCC_ATTR(no_sanitize_undefined)
+GCC_ATTR(no_split_stack)
+GCC_ATTR(no_stack_limit)
+GCC_ATTR(noclone)
+GCC_ATTR(nocommon)
+GCC_ATTR(nocompression)
+GCC_ATTR(nodiscard)
+GCC_ATTR(noinit)
+GCC_ATTR(noinline)
+GCC_ATTR(noipa)
+GCC_ATTR(nomicromips)
+GCC_ATTR(nomips16)
+GCC_ATTR(nonnull)
+GCC_ATTR(noplt)
+GCC_ATTR(noreturn)
+GCC_ATTR(nosave_low_regs)
+GCC_ATTR(not_nested)
+GCC_ATTR(nothrow)
+GCC_ATTR(notshared)
+GCC_ATTR(optimize)
+GCC_ATTR(packed)
+GCC_ATTR(partial_save)
+GCC_ATTR(patchable_function_entry)
+GCC_ATTR(pcs)
+GCC_ATTR(persistent)
+GCC_ATTR(progmem)
+GCC_ATTR(protected)
+GCC_ATTR(pure)
+GCC_ATTR(reentrant)
+GCC_ATTR(regparm)
+GCC_ATTR(renesas)
+GCC_ATTR(resbank)
+GCC_ATTR(reset)
+GCC_ATTR(returns_nonnull)
+GCC_ATTR(returns_twice)
+GCC_ATTR(s390_vector_bool)
+GCC_ATTR(saddr)
+GCC_ATTR(save_all)
+GCC_ATTR(save_volatiles)
+GCC_ATTR(saveall)
+GCC_ATTR(scalar_storage_order)
+GCC_ATTR(sda)
+GCC_ATTR(section)
+GCC_ATTR(selectany)
+GCC_ATTR(sentinel)
+GCC_ATTR(shared)
+GCC_ATTR(short_call)
+GCC_ATTR(shortcall)
+GCC_ATTR(signal)
+GCC_ATTR(simd)
+GCC_ATTR(sp_switch)
+GCC_ATTR(spu_vector)
+GCC_ATTR(sseregparm)
+GCC_ATTR(stack_protect)
+GCC_ATTR(stdcall)
+GCC_ATTR(syscall_linkage)
+GCC_ATTR(sysv_abi)
+GCC_ATTR(target)
+GCC_ATTR(target_clones)
+GCC_ATTR(tda)
+GCC_ATTR(thiscall)
+GCC_ATTR(tiny)
+GCC_ATTR(tiny_data)
+GCC_ATTR(tls_model)
+GCC_ATTR(transaction_callable)
+GCC_ATTR(transaction_may_cancel_outer)
+GCC_ATTR(transaction_pure)
+GCC_ATTR(transaction_safe)
+GCC_ATTR(transaction_safe_dynamic)
+GCC_ATTR(transaction_unsafe)
+GCC_ATTR(transaction_wrap)
+GCC_ATTR(transparent_union)
+GCC_ATTR(trap_exit)
+GCC_ATTR(trapa_handler)
+GCC_ATTR(unused)
+GCC_ATTR(upper)
+GCC_ATTR(use_debug_exception_return)
+GCC_ATTR(use_shadow_register_set)
+GCC_ATTR(used)
+GCC_ATTR(vector)
+GCC_ATTR(vector_size)
+GCC_ATTR(version_id)
+GCC_ATTR(visibility)
+GCC_ATTR(vliw)
+GCC_ATTR(volatile)
+GCC_ATTR(wakeup)
+GCC_ATTR(warm)
+GCC_ATTR(warn_unused)
+GCC_ATTR(warn_unused_result)
+GCC_ATTR(warning)
+GCC_ATTR(weak)
+GCC_ATTR(weakref)
+GCC_ATTR(zda)

dmr_c/src/ident-list.h

@@ -0,0 +1,140 @@
+
+#define IDENT(n) __IDENT(n## _ident, #n, 0)
+#define IDENT_RESERVED(n) __IDENT(n## _ident, #n, 1)
+
+/* Basic C reserved words.. */
+IDENT_RESERVED(sizeof);
+IDENT_RESERVED(if);
+IDENT_RESERVED(else);
+IDENT_RESERVED(return);
+IDENT_RESERVED(switch);
+IDENT_RESERVED(case);
+IDENT_RESERVED(default);
+IDENT_RESERVED(break);
+IDENT_RESERVED(continue);
+IDENT_RESERVED(for);
+IDENT_RESERVED(while);
+IDENT_RESERVED(do);
+IDENT_RESERVED(goto);
+
+/* C typenames. They get marked as reserved when initialized */
+IDENT(struct);
+IDENT(union);
+IDENT(enum);
+IDENT(__attribute); IDENT(__attribute__);
+IDENT(volatile); IDENT(__volatile); IDENT(__volatile__);
+IDENT(double);
+
+/* C storage classes. They get marked as reserved when initialized */
+IDENT(static);
+
+/* C99 keywords */
+IDENT(restrict); IDENT(__restrict); IDENT(__restrict__);
+IDENT(_Bool);
+IDENT_RESERVED(_Complex);
+IDENT_RESERVED(_Imaginary);
+
+/* C11 keywords */
+IDENT(_Alignas);
+IDENT_RESERVED(_Alignof);
+IDENT_RESERVED(_Atomic);
+IDENT_RESERVED(_Generic);
+IDENT(_Noreturn);
+IDENT_RESERVED(_Static_assert);
+IDENT(_Thread_local);
+
+/* Special case for L'\t' */
+IDENT(L);
+
+/* Extended gcc identifiers */
+IDENT(asm); IDENT_RESERVED(__asm); IDENT_RESERVED(__asm__);
+IDENT(alignof); IDENT_RESERVED(__alignof); IDENT_RESERVED(__alignof__); 
+IDENT_RESERVED(__sizeof_ptr__);
+IDENT_RESERVED(__builtin_types_compatible_p);
+IDENT_RESERVED(__builtin_offsetof);
+IDENT_RESERVED(__label__);
+
+/* Attribute names */
+IDENT(packed); IDENT(__packed__);
+IDENT(aligned); IDENT(__aligned__);
+IDENT(nocast);
+IDENT(noderef);
+IDENT(safe);
+IDENT(force);
+IDENT(address_space);
+IDENT(context);
+IDENT(mode); IDENT(__mode__);
+IDENT(QI); IDENT(__QI__);
+IDENT(HI); IDENT(__HI__);
+IDENT(SI); IDENT(__SI__);
+IDENT(DI); IDENT(__DI__);
+IDENT(word); IDENT(__word__);
+IDENT(format); IDENT(__format__);
+IDENT(section); IDENT(__section__);
+IDENT(unused); IDENT(__unused__);
+IDENT(const); IDENT(__const); IDENT(__const__);
+IDENT(used); IDENT(__used__);
+IDENT(warn_unused_result); IDENT(__warn_unused_result__);
+IDENT(noinline); IDENT(__noinline__);
+IDENT(deprecated); IDENT(__deprecated__);
+IDENT(noreturn); IDENT(__noreturn__);
+IDENT(regparm); IDENT(__regparm__);
+IDENT(weak); IDENT(__weak__);
+IDENT(no_instrument_function); IDENT(__no_instrument_function__);
+IDENT(sentinel); IDENT(__sentinel__);
+IDENT(alias); IDENT(__alias__);
+IDENT(pure); IDENT(__pure__);
+IDENT(always_inline); IDENT(__always_inline__);
+IDENT(syscall_linkage); IDENT(__syscall_linkage__);
+IDENT(visibility); IDENT(__visibility__);
+IDENT(bitwise); IDENT(__bitwise__);
+IDENT(model); IDENT(__model__);
+IDENT(format_arg); IDENT(__format_arg__);
+IDENT(nothrow); IDENT(__nothrow); IDENT(__nothrow__);
+IDENT(__transparent_union__);
+IDENT(malloc);
+IDENT(__malloc__);
+IDENT(nonnull); IDENT(__nonnull); IDENT(__nonnull__);
+IDENT(constructor); IDENT(__constructor__);
+IDENT(destructor); IDENT(__destructor__);
+IDENT(cold); IDENT(__cold__);
+IDENT(hot); IDENT(__hot__);
+IDENT(cdecl); IDENT(__cdecl__);
+IDENT(stdcall); IDENT(__stdcall__);
+IDENT(fastcall); IDENT(__fastcall__);
+IDENT(dllimport); IDENT(__dllimport__);
+IDENT(dllexport); IDENT(__dllexport__);
+IDENT(artificial); IDENT(__artificial__);
+IDENT(leaf); IDENT(__leaf__);
+IDENT(vector_size); IDENT(__vector_size__);
+IDENT(error); IDENT(__error__);
+
+
+/* Preprocessor idents.  Direct use of __IDENT avoids mentioning the keyword
+ * itself by name, preventing these tokens from expanding when compiling
+ * sparse. */
+IDENT(defined);
+IDENT(once);
+__IDENT(pragma_ident, "__pragma__", 0);
+__IDENT(__VA_ARGS___ident, "__VA_ARGS__", 0);
+__IDENT(__LINE___ident, "__LINE__", 0);
+__IDENT(__FILE___ident, "__FILE__", 0);
+__IDENT(__DATE___ident, "__DATE__", 0);
+__IDENT(__TIME___ident, "__TIME__", 0);
+__IDENT(__func___ident, "__func__", 0);
+__IDENT(__FUNCTION___ident, "__FUNCTION__", 0);
+__IDENT(__PRETTY_FUNCTION___ident, "__PRETTY_FUNCTION__", 0);
+__IDENT(__COUNTER___ident, "__COUNTER__", 0);
+
+/* Sparse commands */
+IDENT_RESERVED(__context__);
+IDENT_RESERVED(__range__);
+
+/* Magic function names we recognize */
+IDENT(memset); IDENT(memcpy);
+IDENT(copy_to_user); IDENT(copy_from_user);
+IDENT(main);
+
+#undef __IDENT
+#undef IDENT
+#undef IDENT_RESERVED

dmr_c/src/inline.c

@@ -0,0 +1,590 @@
+/*
+ * Sparse - a semantic source parser.
+ *
+ * Copyright (C) 2003 Transmeta Corp.
+ *               2003-2004 Linus Torvalds
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+ /*
+ * This version is part of the dmr_c project.
+ * Copyright (C) 2017 Dibyendu Majumdar
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <port.h>
+#include <lib.h>
+#include <allocate.h>
+#include <token.h>
+#include <parse.h>
+#include <symbol.h>
+#include <expression.h>
+
+static struct expression * dup_expression(struct dmr_C *C, struct expression *expr)
+{
+	struct expression *dup = dmrC_alloc_expression(C, expr->pos, expr->type);
+	*dup = *expr;
+	return dup;
+}
+
+static struct statement * dup_statement(struct dmr_C *C, struct statement *stmt)
+{
+	struct statement *dup = dmrC_alloc_statement(C, stmt->pos, stmt->type);
+	*dup = *stmt;
+	return dup;
+}
+
+static struct symbol *copy_symbol(struct dmr_C *C, struct position pos, struct symbol *sym)
+{
+	if (!sym)
+		return sym;
+	if (sym->ctype.modifiers & (MOD_STATIC | MOD_EXTERN | MOD_TOPLEVEL | MOD_INLINE))
+		return sym;
+	if (!sym->replace) {
+		dmrC_warning(C, pos, "unreplaced symbol '%s'", dmrC_show_ident(C, sym->ident));
+		return sym;
+	}
+	return sym->replace;
+}
+
+static struct symbol_list *copy_symbol_list(struct dmr_C *C, struct symbol_list *src)
+{
+	struct symbol_list *dst = NULL;
+	struct symbol *sym;
+
+	FOR_EACH_PTR(src, sym) {
+		struct symbol *newsym = copy_symbol(C, sym->pos, sym);
+		dmrC_add_symbol(C, &dst, newsym);
+	} END_FOR_EACH_PTR(sym);
+	return dst;
+}
+
+static struct expression * copy_expression(struct dmr_C *C, struct expression *expr)
+{
+	if (!expr)
+		return NULL;
+
+	switch (expr->type) {
+	/*
+	 * EXPR_SYMBOL is the interesting case, we may need to replace the
+	 * symbol to the new copy.
+	 */
+	case EXPR_SYMBOL: {
+		struct symbol *sym = copy_symbol(C, expr->pos, expr->symbol);
+		if (sym == expr->symbol)
+			break;
+		expr = dup_expression(C, expr);
+		expr->symbol = sym;
+		break;
+	}
+
+	/* Atomics, never change, just return the expression directly */
+	case EXPR_VALUE:
+	case EXPR_STRING:
+	case EXPR_FVALUE:
+	case EXPR_TYPE:
+		break;
+
+	/* Unops: check if the subexpression is unique */
+	case EXPR_PREOP:
+	case EXPR_POSTOP: {
+		struct expression *unop = copy_expression(C, expr->unop);
+		if (expr->unop == unop)
+			break;
+		expr = dup_expression(C, expr);
+		expr->unop = unop;
+		break;
+	}
+
+	case EXPR_SLICE: {
+		struct expression *base = copy_expression(C, expr->base);
+		expr = dup_expression(C, expr);
+		expr->base = base;
+		break;
+	}
+
+	/* Binops: copy left/right expressions */
+	case EXPR_BINOP:
+	case EXPR_COMMA:
+	case EXPR_COMPARE:
+	case EXPR_LOGICAL: {
+		struct expression *left = copy_expression(C, expr->left);
+		struct expression *right = copy_expression(C, expr->right);
+		if (left == expr->left && right == expr->right)
+			break;
+		expr = dup_expression(C, expr);
+		expr->left = left;
+		expr->right = right;
+		break;
+	}
+
+	case EXPR_ASSIGNMENT: {
+		struct expression *left = copy_expression(C, expr->left);
+		struct expression *right = copy_expression(C, expr->right);
+		if (expr->op == '=' && left == expr->left && right == expr->right)
+			break;
+		expr = dup_expression(C, expr);
+		expr->left = left;
+		expr->right = right;
+		break;
+	}
+
+	/* Dereference */
+	case EXPR_DEREF: {
+		struct expression *deref = copy_expression(C, expr->deref);
+		expr = dup_expression(C, expr);
+		expr->deref = deref;
+		break;
+	}
+
+	/* Cast/sizeof/__alignof__ */
+	case EXPR_CAST:
+		if (expr->cast_expression->type == EXPR_INITIALIZER) {
+			struct expression *cast = expr->cast_expression;
+			struct symbol *sym = expr->cast_type;
+			expr = dup_expression(C, expr);
+			expr->cast_expression = copy_expression(C, cast);
+			expr->cast_type = dmrC_alloc_symbol(C->S, sym->pos, sym->type);
+			*expr->cast_type = *sym;
+			break;
+		}
+	case EXPR_FORCE_CAST:
+	case EXPR_IMPLIED_CAST:
+	case EXPR_SIZEOF: 
+	case EXPR_PTRSIZEOF:
+	case EXPR_ALIGNOF: {
+		struct expression *cast = copy_expression(C, expr->cast_expression);
+		if (cast == expr->cast_expression)
+			break;
+		expr = dup_expression(C, expr);
+		expr->cast_expression = cast;
+		break;
+	}
+
+	/* Conditional expression */
+	case EXPR_SELECT:
+	case EXPR_CONDITIONAL: {
+		struct expression *cond = copy_expression(C, expr->conditional);
+		struct expression *truee = copy_expression(C, expr->cond_true);
+		struct expression *falsee = copy_expression(C, expr->cond_false);
+		if (cond == expr->conditional && truee == expr->cond_true && falsee == expr->cond_false)
+			break;
+		expr = dup_expression(C, expr);
+		expr->conditional = cond;
+		expr->cond_true = truee;
+		expr->cond_false = falsee;
+		break;
+	}
+
+	/* Statement expression */
+	case EXPR_STATEMENT: {
+		struct statement *stmt = dmrC_alloc_statement(C, expr->pos, STMT_COMPOUND);
+		dmrC_copy_statement(C, expr->statement, stmt);
+		expr = dup_expression(C, expr);
+		expr->statement = stmt;
+		break;
+	}
+
+	/* Call expression */
+	case EXPR_CALL: {
+		struct expression *fn = copy_expression(C, expr->fn);
+		struct expression_list *list = expr->args;
+		struct expression *arg;
+
+		expr = dup_expression(C, expr);
+		expr->fn = fn;
+		expr->args = NULL;
+		FOR_EACH_PTR(list, arg) {
+			dmrC_add_expression(C, &expr->args, copy_expression(C, arg));
+		} END_FOR_EACH_PTR(arg);
+		break;
+	}
+
+	/* Initializer list statement */
+	case EXPR_INITIALIZER: {
+		struct expression_list *list = expr->expr_list;
+		struct expression *entry;
+		expr = dup_expression(C, expr);
+		expr->expr_list = NULL;
+		FOR_EACH_PTR(list, entry) {
+			dmrC_add_expression(C, &expr->expr_list, copy_expression(C, entry));
+		} END_FOR_EACH_PTR(entry);
+		break;
+	}
+
+	/* Label in inline function - hmm. */
+	case EXPR_LABEL: {
+		struct symbol *label_symbol = copy_symbol(C, expr->pos, expr->label_symbol);
+		expr = dup_expression(C, expr);
+		expr->label_symbol = label_symbol;
+		break;
+	}
+
+	case EXPR_INDEX: {
+		struct expression *sub_expr = copy_expression(C, expr->idx_expression);
+		expr = dup_expression(C, expr);
+		expr->idx_expression = sub_expr;
+		break;
+	}
+		
+	case EXPR_IDENTIFIER: {
+		struct expression *sub_expr = copy_expression(C, expr->ident_expression);
+		expr = dup_expression(C, expr);
+		expr->ident_expression = sub_expr;
+		break;
+	}
+
+	/* Position in initializer.. */
+	case EXPR_POS: {
+		struct expression *val = copy_expression(C, expr->init_expr);
+		expr = dup_expression(C, expr);
+		expr->init_expr = val;
+		break;
+	}
+	case EXPR_OFFSETOF: {
+		struct expression *val = copy_expression(C, expr->down);
+		if (expr->op == '.') {
+			if (expr->down != val) {
+				expr = dup_expression(C, expr);
+				expr->down = val;
+			}
+		} else {
+			struct expression *idx = copy_expression(C, expr->index);
+			if (expr->down != val || expr->index != idx) {
+				expr = dup_expression(C, expr);
+				expr->down = val;
+				expr->index = idx;
+			}
+		}
+		break;
+	}
+	default:
+		dmrC_warning(C, expr->pos, "trying to copy expression type %d", expr->type);
+	}
+	return expr;
+}
+
+static struct expression_list *copy_asm_constraints(struct dmr_C *C, struct expression_list *in)
+{
+	struct expression_list *out = NULL;
+	struct expression *expr;
+	int state = 0;
+
+	FOR_EACH_PTR(in, expr) {
+		switch (state) {
+		case 0: /* identifier */
+		case 1: /* constraint */
+			state++;
+			dmrC_add_expression(C, &out, expr);
+			continue;
+		case 2: /* expression */
+			state = 0;
+			dmrC_add_expression(C, &out, copy_expression(C, expr));
+			continue;
+		}
+	} END_FOR_EACH_PTR(expr);
+	return out;
+}
+
+static void set_replace(struct symbol *old, struct symbol *news)
+{
+	news->replace = old;
+	old->replace = news;
+}
+
+static void unset_replace(struct dmr_C *C, struct symbol *sym)
+{
+	struct symbol *r = sym->replace;
+	if (!r) {
+		dmrC_warning(C, sym->pos, "symbol '%s' not replaced?", dmrC_show_ident(C, sym->ident));
+		return;
+	}
+	r->replace = NULL;
+	sym->replace = NULL;
+}
+
+static void unset_replace_list(struct dmr_C *C, struct symbol_list *list)
+{
+	struct symbol *sym;
+	FOR_EACH_PTR(list, sym) {
+		unset_replace(C, sym);
+	} END_FOR_EACH_PTR(sym);
+}
+
+static struct statement *copy_one_statement(struct dmr_C *C, struct statement *stmt)
+{
+	if (!stmt)
+		return NULL;
+	switch(stmt->type) {
+	case STMT_NONE:
+		break;
+	case STMT_DECLARATION: {
+		struct symbol *sym;
+		struct statement *newstmt = dup_statement(C, stmt);
+		newstmt->declaration = NULL;
+		FOR_EACH_PTR(stmt->declaration, sym) {
+			struct symbol *newsym = copy_symbol(C, stmt->pos, sym);
+			if (newsym != sym)
+				newsym->initializer = copy_expression(C, sym->initializer);
+			dmrC_add_symbol(C, &newstmt->declaration, newsym);
+		} END_FOR_EACH_PTR(sym);
+		stmt = newstmt;
+		break;
+	}
+	case STMT_CONTEXT:
+	case STMT_EXPRESSION: {
+		struct expression *expr = copy_expression(C, stmt->expression);
+		if (expr == stmt->expression)
+			break;
+		stmt = dup_statement(C, stmt);
+		stmt->expression = expr;
+		break;
+	}
+	case STMT_RANGE: {
+		struct expression *expr = copy_expression(C, stmt->range_expression);
+		if (expr == stmt->expression)
+			break;
+		stmt = dup_statement(C, stmt);
+		stmt->range_expression = expr;
+		break;
+	}
+	case STMT_COMPOUND: {
+		struct statement *newst = dmrC_alloc_statement(C, stmt->pos, STMT_COMPOUND);
+		dmrC_copy_statement(C, stmt, newst);
+		stmt = newst;
+		break;
+	}
+	case STMT_IF: {
+		struct expression *cond = stmt->if_conditional;
+		struct statement *trues = stmt->if_true;
+		struct statement *falses = stmt->if_false;
+
+		cond = copy_expression(C, cond);
+		trues = copy_one_statement(C, trues);
+		falses = copy_one_statement(C, falses);
+		if (stmt->if_conditional == cond &&
+		    stmt->if_true == trues &&
+		    stmt->if_false == falses)
+			break;
+		stmt = dup_statement(C, stmt);
+		stmt->if_conditional = cond;
+		stmt->if_true = trues;
+		stmt->if_false = falses;
+		break;
+	}
+	case STMT_RETURN: {
+		struct expression *retval = copy_expression(C, stmt->ret_value);
+		struct symbol *sym = copy_symbol(C, stmt->pos, stmt->ret_target);
+
+		stmt = dup_statement(C, stmt);
+		stmt->ret_value = retval;
+		stmt->ret_target = sym;
+		break;
+	}
+	case STMT_CASE: {
+		stmt = dup_statement(C, stmt);
+		stmt->case_label = copy_symbol(C, stmt->pos, stmt->case_label);
+		stmt->case_label->stmt = stmt;
+		stmt->case_expression = copy_expression(C, stmt->case_expression);
+		stmt->case_to = copy_expression(C, stmt->case_to);
+		stmt->case_statement = copy_one_statement(C, stmt->case_statement);
+		break;
+	}
+	case STMT_SWITCH: {
+		struct symbol *switch_break = copy_symbol(C, stmt->pos, stmt->switch_break);
+		struct symbol *switch_case = copy_symbol(C, stmt->pos, stmt->switch_case);
+		struct expression *expr = copy_expression(C, stmt->switch_expression);
+		struct statement *switch_stmt = copy_one_statement(C, stmt->switch_statement);
+
+		stmt = dup_statement(C, stmt);
+		switch_case->symbol_list = copy_symbol_list(C, switch_case->symbol_list);
+		stmt->switch_break = switch_break;
+		stmt->switch_case = switch_case;
+		stmt->switch_expression = expr;
+		stmt->switch_statement = switch_stmt;
+		break;		
+	}
+	case STMT_ITERATOR: {
+		stmt = dup_statement(C, stmt);
+		stmt->iterator_break = copy_symbol(C, stmt->pos, stmt->iterator_break);
+		stmt->iterator_continue = copy_symbol(C, stmt->pos, stmt->iterator_continue);
+		stmt->iterator_syms = copy_symbol_list(C, stmt->iterator_syms);
+
+		stmt->iterator_pre_statement = copy_one_statement(C, stmt->iterator_pre_statement);
+		stmt->iterator_pre_condition = copy_expression(C, stmt->iterator_pre_condition);
+
+		stmt->iterator_statement = copy_one_statement(C, stmt->iterator_statement);
+
+		stmt->iterator_post_statement = copy_one_statement(C, stmt->iterator_post_statement);
+		stmt->iterator_post_condition = copy_expression(C, stmt->iterator_post_condition);
+		break;
+	}
+	case STMT_LABEL: {
+		stmt = dup_statement(C, stmt);
+		stmt->label_identifier = copy_symbol(C, stmt->pos, stmt->label_identifier);
+		stmt->label_statement = copy_one_statement(C, stmt->label_statement);
+		break;
+	}
+	case STMT_GOTO: {
+		stmt = dup_statement(C, stmt);
+		stmt->goto_label = copy_symbol(C, stmt->pos, stmt->goto_label);
+		stmt->goto_expression = copy_expression(C, stmt->goto_expression);
+		stmt->target_list = copy_symbol_list(C, stmt->target_list);
+		break;
+	}
+	case STMT_ASM: {
+		stmt = dup_statement(C, stmt);
+		stmt->asm_inputs = copy_asm_constraints(C, stmt->asm_inputs);
+		stmt->asm_outputs = copy_asm_constraints(C, stmt->asm_outputs);
+		/* no need to dup "clobbers", since they are all constant strings */
+		break;
+	}
+	default:
+		dmrC_warning(C, stmt->pos, "trying to copy statement type %d", stmt->type);
+		break;
+	}
+	return stmt;
+}
+
+/*
+ * Copy a statement tree from 'src' to 'dst', where both
+ * source and destination are of type STMT_COMPOUND.
+ *
+ * We do this for the tree-level inliner.
+ *
+ * This doesn't do the symbol replacement right: it's not
+ * re-entrant.
+ */
+void dmrC_copy_statement(struct dmr_C *C, struct statement *src, struct statement *dst)
+{
+	struct statement *stmt;
+
+	FOR_EACH_PTR(src->stmts, stmt) {
+		dmrC_add_statement(C, &dst->stmts, copy_one_statement(C, stmt));
+	} END_FOR_EACH_PTR(stmt);
+	dst->args = copy_one_statement(C, src->args);
+	dst->ret = copy_symbol(C, src->pos, src->ret);
+	dst->inline_fn = src->inline_fn;
+}
+
+static struct symbol *create_copy_symbol(struct dmr_C *C, struct symbol *orig)
+{
+	struct symbol *sym = orig;
+	if (orig) {
+		sym = dmrC_alloc_symbol(C->S, orig->pos, orig->type);
+		*sym = *orig;
+		sym->bb_target = NULL;
+		sym->pseudo = NULL;
+		set_replace(orig, sym);
+		orig = sym;
+	}
+	return orig;
+}
+
+static struct symbol_list *create_symbol_list(struct dmr_C *C, struct symbol_list *src)
+{
+	struct symbol_list *dst = NULL;
+	struct symbol *sym;
+
+	FOR_EACH_PTR(src, sym) {
+		struct symbol *newsym = create_copy_symbol(C, sym);
+		dmrC_add_symbol(C, &dst, newsym);
+	} END_FOR_EACH_PTR(sym);
+	return dst;
+}
+
+int dmrC_inline_function(struct dmr_C *C, struct expression *expr, struct symbol *sym)
+{
+	struct symbol_list * fn_symbol_list;
+	struct symbol *fn = sym->ctype.base_type;
+	struct expression_list *arg_list = expr->args;
+	struct statement *stmt = dmrC_alloc_statement(C, expr->pos, STMT_COMPOUND);
+	struct symbol_list *name_list, *arg_decl;
+	struct symbol *name;
+	struct expression *arg;
+
+	if (!fn->inline_stmt) {
+		dmrC_sparse_error(C, fn->pos, "marked inline, but without a definition");
+		return 0;
+	}
+	if (fn->expanding)
+		return 0;
+
+	fn->expanding = 1;
+
+	name_list = fn->arguments;
+
+	expr->type = EXPR_STATEMENT;
+	expr->statement = stmt;
+	expr->ctype = fn->ctype.base_type;
+
+	fn_symbol_list = create_symbol_list(C, sym->inline_symbol_list);
+
+	arg_decl = NULL;
+	PREPARE_PTR_LIST(name_list, name);
+	FOR_EACH_PTR(arg_list, arg) {
+		struct symbol *a = dmrC_alloc_symbol(C->S, arg->pos, SYM_NODE);
+
+		a->ctype.base_type = arg->ctype;
+		if (name) {
+			*a = *name;
+			set_replace(name, a);
+			dmrC_add_symbol(C, &fn_symbol_list, a);
+		}
+		a->initializer = arg;
+		dmrC_add_symbol(C, &arg_decl, a);
+
+		NEXT_PTR_LIST(name);
+	} END_FOR_EACH_PTR(arg);
+	FINISH_PTR_LIST(name);
+
+	dmrC_copy_statement(C, fn->inline_stmt, stmt);
+
+	if (arg_decl) {
+		struct statement *decl = dmrC_alloc_statement(C, expr->pos, STMT_DECLARATION);
+		decl->declaration = arg_decl;
+		stmt->args = decl;
+	}
+	stmt->inline_fn = sym;
+
+	unset_replace_list(C, fn_symbol_list);
+
+	dmrC_evaluate_statement(C, stmt);
+
+	fn->expanding = 0;
+	return 1;
+}
+
+void dmrC_uninline(struct dmr_C *C, struct symbol *sym)
+{
+	struct symbol *fn = sym->ctype.base_type;
+	struct symbol_list *arg_list = fn->arguments;
+	struct symbol *p;
+
+	sym->symbol_list = create_symbol_list(C, sym->inline_symbol_list);
+	FOR_EACH_PTR(arg_list, p) {
+		p->replace = p;
+	} END_FOR_EACH_PTR(p);
+	fn->stmt = dmrC_alloc_statement(C, fn->pos, STMT_COMPOUND);
+	dmrC_copy_statement(C, fn->inline_stmt, fn->stmt);
+	unset_replace_list(C, sym->symbol_list);
+	unset_replace_list(C, arg_list);
+}

dmr_c/src/lib.h

@@ -0,0 +1,302 @@
+#ifndef DMR_LIB_H
+#define DMR_LIB_H
+
+/* This file is derived from lib.h in sparse */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <time.h>
+#include <setjmp.h>
+
+/*
+* Basic helper routine descriptions for 'sparse'.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*               2004 Christopher Li
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+// Build options
+#define NEW_SSA 0
+#define SINGLE_STORE_SHORTCUT 1
+
+
+#include <allocate.h>
+#include <ptrlist.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define DO_STRINGIFY(x) #x
+#define STRINGIFY(x) DO_STRINGIFY(x)
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+
+#define MAX_STRING 8191
+
+extern unsigned int dmrC_hexval(unsigned int c);
+
+struct position {
+	unsigned int type : 6, 
+		stream : 14, 
+		newline : 1, 
+		whitespace : 1,
+		pos : 10;
+	unsigned int line : 31, 
+		noexpand : 1;
+};
+
+struct ident;
+struct token;
+struct symbol;
+struct statement;
+struct expression;
+struct basic_block;
+struct entrypoint;
+struct instruction;
+struct multijmp;
+struct pseudo;
+struct string;
+typedef struct pseudo *pseudo_t;
+
+struct target_t;
+struct global_symbols_t;
+struct tokenizer_state_t;
+struct linearizer_state_t;
+
+struct warning {
+	const char *name;
+	int *flag;
+};
+
+enum standard {
+	STANDARD_C89,
+	STANDARD_C94,
+	STANDARD_C99,
+	STANDARD_C11,
+	STANDARD_GNU11,
+	STANDARD_GNU89,
+	STANDARD_GNU99,
+};
+
+enum {
+	WARNING_OFF,
+	WARNING_ON,
+	WARNING_FORCE_OFF
+};
+
+struct symbol_list;
+struct statement_list;
+struct expression_list;
+struct basic_block_list;
+struct instruction_list;
+struct multijmp_list;
+struct pseudo_list;
+struct phi_map;
+
+DECLARE_PTR_LIST(string_list, char);
+
+#define   ERROR_CURR_PHASE        (1 << 0)
+#define   ERROR_PREV_PHASE        (1 << 1)
+
+struct dmr_C {
+	struct target_t *target;
+	struct global_symbols_t *S;
+	struct tokenizer_state_t *T;
+	struct parse_state_t *P;
+	struct linearizer_state_t *L;
+
+    void *User_data;
+
+	// memory allocators
+	struct allocator ptrlist_allocator;
+	struct allocator token_allocator;
+	struct allocator protected_token_allocator;
+	struct allocator byte_allocator;
+	struct allocator string_allocator;
+	struct allocator ident_allocator;
+	struct allocator scope_allocator;
+	struct allocator expression_allocator;
+	struct allocator statement_allocator;
+
+	int max_warnings;
+	int show_info;
+	int errors;
+	int die_if_error;
+	int once;
+	int preprocess_only;
+	int codegen;
+	int has_error;
+
+	jmp_buf jmpbuf;
+
+	const char *gcc_base_dir;
+	const char *multiarch_dir;
+
+	int verbose, optimize, optimize_size, preprocessing;
+
+	enum standard standard;
+	struct token *pre_buffer_begin;
+	struct token *pre_buffer_end;
+
+	int Waddress; // TODO
+	int Waddress_space;
+	int Wbitwise;
+	int Wcast_to_as;
+	int Wcast_truncate;
+	int Wcontext;
+	int Wdecl;
+	int Wdeclarationafterstatement;
+	int Wdefault_bitfield_sign;
+	int Wdesignated_init;
+	int Wdo_while;
+	int Wenum_mismatch;
+	int Wsparse_error;
+	int Winit_cstring;
+	int Wmemcpy_max_count; // TODO
+	int Wnon_pointer_null;
+	int Wold_initializer;
+	int Wone_bit_signed_bitfield;
+	int Woverride_init; //TODO
+	int Woverride_init_all; //TODO
+	int Woverride_init_whole_range; //TODO
+	int Wparen_string;
+	int Wptr_subtraction_blows;
+	int Wreturn_void;
+	int Wshadow;
+	int Wsizeof_bool;
+	int Wtautological_compare;
+	int Wtransparent_union;
+	int Wtypesign;
+	int Wundef;
+	int Wuninitialized;
+	int Wunknown_attribute;
+	int Wvla;
+	struct warning warnings[32];
+	struct warning debugs[2];
+	struct warning dumps[1];
+
+#define CMDLINE_INCLUDE 20
+	int cmdline_include_nr;
+	char *cmdline_include[CMDLINE_INCLUDE];
+
+	int dump_macro_defs; // TODO
+
+	int dbg_entry;
+	int dbg_dead;
+	int fmem_report;	// TODO
+	int fdump_linearize;	// TODO
+	unsigned long long fmemcpy_max_count;
+
+	int arch_m64;
+	int arch_msize_long;
+	int arch_big_endian;	// TODO
+	/* TODO is this the right place? */
+	struct scope *block_scope, *function_scope, *file_scope, *global_scope;
+	struct scope *builtin_scope;
+
+	/* Current parsing/evaluation function */
+	struct symbol *current_fn;
+
+	char modifier_string_buffer[100];
+	char typename_array[200];
+
+	struct ident_list *macros;	// only needed for -dD
+	int false_nesting;
+	int counter_macro;		// __COUNTER__ expansion
+
+#define INCLUDEPATHS 300
+	const char *includepath[INCLUDEPATHS + 1];
+
+	const char **quote_includepath;
+	const char **angle_includepath;
+	const char **isys_includepath;
+	const char **sys_includepath;
+	const char **dirafter_includepath;
+
+	char date_buffer[12]; /* __DATE__: 3 + ' ' + 2 + ' ' + 4 + '\0' */
+	char preprocessor_buffer[MAX_STRING];
+	char preprocessor_mergebuffer[512];
+	char preprocessor_tokenseqbuffer[256];
+	time_t t;
+	char fullname[1024];
+
+	char output_file_name[1024];
+};
+
+/*
+* Creates a new instance of dmr_C. Due to the way the parser and compiler works
+* at present it is recommended that each dmr_C instance be used to process one set
+* of inputs only. Destroy the dmr_C instance after use. This way all resources will
+* be released.
+*/
+extern struct dmr_C *new_dmr_C();
+extern void destroy_dmr_C(struct dmr_C *C);
+
+/*
+* Appends the provided formatted string to the "pre buffer" that is processed by
+* dmrC_sparse_initialize(). The input is tokenized immediately and added to the "pre buffer"
+* token stream.
+*/
+extern void dmrC_add_pre_buffer(struct dmr_C *, const char *fmt, ...) FORMAT_ATTR(2);
+
+/*
+* Declares a bunch of gcc built-ins into a "pre buffer" which is processed in
+* dmrC_sparse_initialize(). The dmrC_add_pre_buffer() function is used to add input into the
+* pre buffer.
+*/
+extern void dmrC_declare_builtin_functions(struct dmr_C *C);
+extern void dmrC_create_builtin_stream(struct dmr_C *C);
+extern void dmrC_dump_macro_definitions(struct dmr_C *C);
+extern struct symbol_list * dmrC_sparse_initialize(struct dmr_C *C, int argc, char **argv, struct string_list **filelist);
+extern struct symbol_list * dmrC_sparse_keep_tokens(struct dmr_C *C, char *filename);
+extern struct symbol_list * dmrC_sparse(struct dmr_C *C, char *filename);
+extern struct symbol_list * dmrC__sparse(struct dmr_C *C, char *filename);
+extern struct symbol_list * dmrC_sparse_buffer(struct dmr_C *C, const char *name, char *buffer, int keep_tokens);
+
+struct token *dmrC_skip_to_token(struct token *, int);
+struct token *dmrC_expect_token(struct dmr_C *C, struct token *token, int op, const char *where);
+
+extern void dmrC_die(struct dmr_C *, const char *, ...) FORMAT_ATTR(2) NORETURN_ATTR;
+extern void dmrC_info(struct dmr_C *, struct position, const char *, ...)
+    FORMAT_ATTR(3);
+extern void dmrC_warning(struct dmr_C *, struct position, const char *, ...)
+    FORMAT_ATTR(3);
+extern void dmrC_sparse_error(struct dmr_C *, struct position, const char *, ...)
+    FORMAT_ATTR(3);
+extern void dmrC_error_die(struct dmr_C *, struct position, const char *, ...)
+    FORMAT_ATTR(3) NORETURN_ATTR;
+extern void dmrC_expression_error(struct dmr_C *, struct expression *, const char *,
+			     ...) FORMAT_ATTR(3);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/linearize.h

@@ -0,0 +1,506 @@
+#ifndef DMR_C_LINEARIZE_H
+#define DMR_C_LINEARIZE_H
+
+/*
+* Linearize - walk the parse tree and generate a linear version
+* of it and the basic blocks.
+*
+* Copyright (C) 2004 Linus Torvalds
+* Copyright (C) 2004 Christopher Li
+*/
+
+#include <lib.h>
+#include <allocate.h>
+#include <token.h>
+#include <parse.h>
+#include <symbol.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct instruction;
+
+DECLARE_PTR_LIST(basic_block_list, struct basic_block);
+DECLARE_PTR_LIST(instruction_list, struct instruction);
+DECLARE_PTR_LIST(multijmp_list, struct multijmp);
+DECLARE_PTR_LIST(pseudo_list, struct pseudo);
+
+struct pseudo_user {
+	struct instruction *insn;
+	pseudo_t *userp;
+};
+
+DECLARE_PTR_LIST(pseudo_user_list, struct pseudo_user);
+
+enum pseudo_type {
+	PSEUDO_VOID,
+	PSEUDO_REG,
+	PSEUDO_SYM,
+	PSEUDO_VAL,
+	PSEUDO_ARG,
+	PSEUDO_PHI,
+};
+
+/* Have you ever heard of "static single assignment" or SSA form? 
+   struct pseudo represents one of those single-assignment variables. 
+   Each one has a pointer to the symbol it represents (which may 
+   have many pseudos referencing it). Each one also has a pointer 
+   to the instruction that defines it.*/
+struct pseudo {
+	int nr;
+	enum pseudo_type type;
+	int size; /* OP_SETVAL only */
+	struct pseudo_user_list *users; /* pseudo_user list */
+	struct ident *ident;
+	union {
+		struct symbol *sym;	     // PSEUDO_SYM, VAL & ARG
+		struct instruction *def; // PSEUDO_REG & PHI
+		long long value;	     // PSEUDO_VAL
+	};
+	DMRC_BACKEND_TYPE priv;
+	DMRC_BACKEND_TYPE priv2;	/* FIXME - we use this to save ptr to allocated stack in PHI instructions (nanojit) */
+};
+
+struct linearizer_state_t {
+	struct allocator pseudo_allocator;
+	struct allocator pseudo_user_allocator;
+	struct allocator asm_constraint_allocator;
+	struct allocator asm_rules_allocator;
+	struct allocator multijmp_allocator;
+	struct allocator basic_block_allocator;
+	struct allocator entrypoint_allocator;
+	struct allocator instruction_allocator;
+
+	struct pseudo void_pseudo;
+	struct position current_pos;
+
+	int repeat_phase;
+	unsigned long bb_generation;
+	int liveness_changed;
+
+	struct pseudo_list **live_list;
+	struct pseudo_list *dead_list;
+
+#define MAX_VAL_HASH 64
+	struct pseudo_list *prev[MAX_VAL_HASH]; /* from pseudo_t value_pseudo(long long val) in linearize.c */
+	int nr;	/* pseudo number */
+	int bb_nr; /* basic block number */
+	char buffer[4096*4];
+	int n;
+	char pseudo_buffer[4][64];
+#define INSN_HASH_SIZE 256
+	struct instruction_list *insn_hash_table[INSN_HASH_SIZE];
+};
+
+#define VOID_PSEUDO(C) (&C->L->void_pseudo)
+
+struct multijmp {
+	struct basic_block *target;
+	long long begin, end;
+};
+
+struct asm_constraint {
+	pseudo_t pseudo;
+	const char *constraint;
+	const struct ident *ident;
+};
+
+DECLARE_PTR_LIST(asm_constraint_list, struct asm_constraint);
+
+struct asm_rules {
+	struct asm_constraint_list *inputs; /* list of asm_constraint */
+	struct asm_constraint_list *outputs; /* list of asm_constraint */
+	struct asm_constraint_list *clobbers; /* list of asm_constraint */
+};
+
+struct instruction {
+	unsigned opcode:8,
+		 size:24;
+	struct basic_block *bb;
+	struct position pos;
+	struct symbol *type;
+	union {
+		pseudo_t target;
+		pseudo_t cond;		/* for branch and switch */
+	};
+	union {
+		struct /* entrypoint */ {
+			struct pseudo_list *arg_list; /* pseudo list */
+		};
+		struct /* branch */ {
+			struct basic_block *bb_true, *bb_false;
+		};
+		struct /* switch */ {
+			struct multijmp_list *multijmp_list;
+		};
+		struct /* phi_node */ {
+			struct pseudo_list *phi_list; /* pseudo list */
+		};
+		struct /* phi source */ {
+			pseudo_t phi_src;
+			struct instruction_list *phi_users; /* instruction list */
+		};
+		struct /* unops */ {
+			pseudo_t src;
+			struct symbol *orig_type;	/* casts */
+			unsigned int offset;		/* memops */
+		};
+		struct /* binops and sel */ {
+			pseudo_t src1, src2, src3;
+		};
+		struct /* slice */ {
+			pseudo_t base;
+			unsigned from, len;
+		};
+		struct /* setval */ {
+			pseudo_t symbol;		/* Subtle: same offset as "src" !! */
+			struct expression *val;
+		};
+		struct /* call */ {
+			pseudo_t func;
+			struct pseudo_list *arguments; /* instruction list */
+			struct symbol *fntype;
+		};
+		struct /* context */ {
+			int increment;
+			int check;
+			struct expression *context_expr;
+		};
+		struct /* asm */ {
+			const char *string;
+			struct asm_rules *asm_rules;
+		};
+	};
+};
+
+enum opcode {
+	OP_BADOP,
+
+	/* Entry */
+	OP_ENTRY,
+
+	/* Terminator */
+	OP_TERMINATOR,
+	OP_RET = OP_TERMINATOR,
+	OP_BR,
+	OP_CBR,
+	OP_SWITCH,
+	OP_INVOKE,
+	OP_COMPUTEDGOTO,
+	OP_UNWIND,
+	OP_TERMINATOR_END = OP_UNWIND,
+	
+	/* Binary */
+	OP_BINARY,
+	OP_ADD = OP_BINARY,
+	OP_SUB,
+	OP_MULU, OP_MULS,
+	OP_DIVU, OP_DIVS,
+	OP_MODU, OP_MODS,
+	OP_SHL,
+	OP_LSR, OP_ASR,
+	
+	/* Logical */
+	OP_AND,
+	OP_OR,
+	OP_XOR,
+	OP_AND_BOOL,
+	OP_OR_BOOL,
+	OP_BINARY_END = OP_OR_BOOL,
+
+	/* Binary comparison */
+	OP_BINCMP,
+	OP_SET_EQ = OP_BINCMP,
+	OP_SET_NE,
+	OP_SET_LE,
+	OP_SET_GE,
+	OP_SET_LT,
+	OP_SET_GT,
+	OP_SET_B,
+	OP_SET_A,
+	OP_SET_BE,
+	OP_SET_AE,
+	OP_BINCMP_END = OP_SET_AE,
+
+	/* Uni */
+	OP_NOT,
+	OP_NEG,
+
+	/* Select - three input values */
+	OP_SEL,
+	
+	/* Memory */
+	OP_MALLOC,
+	OP_FREE,
+	OP_ALLOCA,
+	OP_LOAD,
+	OP_STORE,
+	OP_SETVAL,
+	OP_SYMADDR,
+	OP_GET_ELEMENT_PTR,
+
+	/* Other */
+	OP_PHI,
+	OP_PHISOURCE,
+	OP_CAST,
+	OP_SCAST,
+	OP_FPCAST,
+	OP_PTRCAST,
+	OP_INLINED_CALL,
+	OP_CALL,
+	OP_VANEXT,
+	OP_VAARG,
+	OP_SLICE,
+	OP_SNOP,
+	OP_LNOP,
+	OP_NOP,
+	OP_DEATHNOTE,
+	OP_ASM,
+
+	/* Sparse tagging (line numbers, context, whatever) */
+	OP_CONTEXT,
+	OP_RANGE,
+
+	/* Needed to translate SSA back to normal form */
+	OP_COPY,
+};
+
+/*
+A basic block represents a series of instructions with no branches. 
+Straight-line code. A branch only occurs at the end of a basic block, 
+and branches can only target the beginning of a basic block. Typically, 
+a conditional will consist of a basic block leading up to the branch, 
+a basic block for the true case, a basic block for the false case, 
+and a basic block where the two paths merge back together. Either the true 
+or the false case may not exist. A loop will normally have a basic block 
+for the loop body, which can branch to the top at the end or continue 
+to the next basic block. So basic blocks represent a node in the control 
+flow graph. The edges in that graph lead from one basic block to a 
+basic block which can follow it in the execution of the program.
+*/
+struct basic_block {
+	struct position pos;
+	unsigned long generation;
+	int context;
+	struct entrypoint *ep;
+	struct basic_block_list *parents; /* basic_block sources */ /* predecessors */
+	struct basic_block_list *children; /* basic_block destinations */ /* successors */
+	struct instruction_list *insns;	/* Linear list of instructions */
+	struct pseudo_list *needs, *defines; /* pseudo lists */
+    /* TODO Following fields are used by the codegen backends.
+       In Sparse this is a union but we need the nr field 
+       for NanoJIT backend's liveness analysis in addition to 
+       creating unique labels.
+    */
+	//union {
+		unsigned int nr;	/* unique id for label's names */
+		DMRC_BACKEND_TYPE priv;
+	//};
+};
+
+static inline int dmrC_instruction_list_size(struct instruction_list *list)
+{
+	return ptrlist_size((struct ptr_list *)list);
+}
+
+static inline int dmrC_pseudo_list_size(struct pseudo_list *list)
+{
+	return ptrlist_size((struct ptr_list *)list);
+}
+
+static inline int dmrC_bb_list_size(struct basic_block_list *list)
+{
+	return ptrlist_size((struct ptr_list *)list);
+}
+
+static inline void dmrC_free_instruction_list(struct pseudo_list **head)
+{
+	ptrlist_remove_all((struct ptr_list **)head);
+}
+
+static inline struct instruction * dmrC_delete_last_instruction(struct instruction_list **head)
+{
+	return (struct instruction *) ptrlist_undo_last((struct ptr_list **)head);
+}
+
+static inline struct basic_block * dmrC_delete_last_basic_block(struct basic_block_list **head)
+{
+	return (struct basic_block *) ptrlist_delete_last((struct ptr_list **)head);
+}
+
+static inline struct basic_block *dmrC_first_basic_block(struct basic_block_list *head)
+{
+	return (struct basic_block *) ptrlist_first((struct ptr_list *)head);
+}
+
+static inline struct instruction *dmrC_last_instruction(struct instruction_list *head)
+{
+	return (struct instruction *) ptrlist_last((struct ptr_list *)head);
+}
+
+static inline struct instruction *dmrC_first_instruction(struct instruction_list *head)
+{
+	return (struct instruction *) ptrlist_first((struct ptr_list *)head);
+}
+
+static inline pseudo_t dmrC_first_pseudo(struct pseudo_list *head)
+{
+	return (pseudo_t) ptrlist_first((struct ptr_list *)head);
+}
+
+static inline void dmrC_concat_basic_block_list(struct basic_block_list *from, struct basic_block_list **to)
+{
+	ptrlist_concat((struct ptr_list *)from, (struct ptr_list **)to);
+}
+
+static inline void dmrC_concat_instruction_list(struct instruction_list *from, struct instruction_list **to)
+{
+	ptrlist_concat((struct ptr_list *)from, (struct ptr_list **)to);
+}
+
+static inline int dmrC_is_branch_goto(struct instruction *br)
+{
+	return br && br->opcode==OP_BR && (!br->bb_true || !br->bb_false);
+}
+
+static inline void dmrC_add_bb(struct dmr_C *C, struct basic_block_list **list, struct basic_block *bb)
+{
+	ptrlist_add((struct ptr_list **)list, bb, &C->ptrlist_allocator);
+}
+
+static inline void dmrC_add_instruction(struct dmr_C *C, struct instruction_list **list, struct instruction *insn)
+{
+	ptrlist_add((struct ptr_list **)list, insn, &C->ptrlist_allocator);
+}
+
+static inline void dmrC_add_multijmp(struct dmr_C *C, struct multijmp_list **list, struct multijmp *multijmp)
+{
+	ptrlist_add((struct ptr_list **)list, multijmp, &C->ptrlist_allocator);
+}
+
+static inline pseudo_t *dmrC_add_pseudo(struct dmr_C *C, struct pseudo_list **list, pseudo_t pseudo)
+{
+	return (pseudo_t *) ptrlist_add((struct ptr_list **)list, pseudo, &C->ptrlist_allocator);
+}
+
+static inline int dmrC_remove_pseudo(struct pseudo_list **list, pseudo_t pseudo)
+{
+	return ptrlist_remove((struct ptr_list **)list, pseudo, 0) != 0;
+}
+
+static inline int dmrC_bb_terminated(struct basic_block *bb)
+{
+	struct instruction *insn;
+	if (!bb)
+		return 0;
+	insn = dmrC_last_instruction(bb->insns);
+	return insn && insn->opcode >= OP_TERMINATOR
+	            && insn->opcode <= OP_TERMINATOR_END;
+}
+
+static inline int dmrC_bb_reachable(struct basic_block *bb)
+{
+	return bb != NULL;
+}
+
+
+static inline void dmrC_add_pseudo_user_ptr(struct dmr_C *C, struct pseudo_user *user, struct pseudo_user_list **list)
+{
+	ptrlist_add((struct ptr_list **)list, user, &C->ptrlist_allocator);
+}
+
+static inline int dmrC_has_use_list(pseudo_t p)
+{
+	return (p && p->type != PSEUDO_VOID && p->type != PSEUDO_VAL);
+}
+
+static inline struct pseudo_user *dmrC_alloc_pseudo_user(struct dmr_C *C, struct instruction *insn, pseudo_t *pp)
+{
+	struct pseudo_user *user = (struct pseudo_user *) dmrC_allocator_allocate(&C->L->pseudo_user_allocator, 0);
+	user->userp = pp;
+	user->insn = insn;
+	return user;
+}
+
+static inline void dmrC_use_pseudo(struct dmr_C *C, struct instruction *insn, pseudo_t p, pseudo_t *pp)
+{
+	*pp = p;
+	if (dmrC_has_use_list(p))
+		dmrC_add_pseudo_user_ptr(C, dmrC_alloc_pseudo_user(C, insn, pp), &p->users);
+}
+
+static inline void dmrC_remove_bb_from_list(struct basic_block_list **list, struct basic_block *entry, int count)
+{
+	ptrlist_remove((struct ptr_list **)list, entry, count);
+}
+
+static inline void dmrC_replace_bb_in_list(struct basic_block_list **list,
+	struct basic_block *old, struct basic_block *newbb, int count)
+{
+	ptrlist_replace((struct ptr_list **)list, old, newbb, count);
+}
+
+struct entrypoint {
+	struct symbol *name;
+	struct symbol_list *syms; /* symbol list */
+	struct pseudo_list *accesses; /* pseudo list */
+	struct basic_block_list *bbs; /* basic_block list */
+	struct basic_block *active;
+	struct instruction *entry;
+};
+
+extern void dmrC_insert_select(struct dmr_C *C, struct basic_block *bb, struct instruction *br, struct instruction *phi, pseudo_t if_true, pseudo_t if_false);
+extern void dmrC_insert_branch(struct dmr_C *C, struct basic_block *bb, struct instruction *br, struct basic_block *target);
+// From Luc: sssa-mini
+struct instruction *dmrC_alloc_phisrc(struct dmr_C *C, pseudo_t pseudo, struct symbol *type);
+pseudo_t dmrC_insert_phi_node(struct dmr_C *C, struct basic_block *bb, struct symbol *type);
+pseudo_t dmrC_alloc_phi(struct dmr_C *C, struct basic_block *source, pseudo_t pseudo, struct symbol *type);
+pseudo_t dmrC_alloc_pseudo(struct dmr_C *C, struct instruction *def);
+pseudo_t dmrC_value_pseudo(struct dmr_C *C, struct symbol *type, long long val);
+unsigned int dmrC_value_size(long long value);
+
+struct entrypoint *dmrC_linearize_symbol(struct dmr_C *C, struct symbol *sym);
+int dmrC_unssa(struct dmr_C *C, struct entrypoint *ep);
+void dmrC_show_entry(struct dmr_C *C, struct entrypoint *ep);
+const char *dmrC_show_pseudo(struct dmr_C *C, pseudo_t pseudo);
+void dmrC_show_bb(struct dmr_C *C, struct basic_block *bb);
+const char *dmrC_show_instruction(struct dmr_C *C, struct instruction *insn);
+
+void dmrC_convert_instruction_target(struct dmr_C *C, struct instruction *insn, pseudo_t src);
+void dmrC_kill_use(struct dmr_C *C, pseudo_t *usep);
+void dmrC_kill_insn(struct dmr_C *C, struct instruction *, int force);
+void dmrC_kill_unreachable_bbs(struct dmr_C *C, struct entrypoint *ep);
+void dmrC_kill_bb(struct dmr_C *C, struct basic_block *);
+
+static inline void dmrC_kill_instruction(struct dmr_C *C, struct instruction *insn)
+{
+	dmrC_kill_insn(C, insn, 0);
+}
+static inline void dmrC_kill_instruction_force(struct dmr_C *C, struct instruction *insn)
+{
+	dmrC_kill_insn(C, insn, 1);
+}
+
+void dmrC_clear_liveness(struct entrypoint *ep);
+void dmrC_track_pseudo_liveness(struct dmr_C *C, struct entrypoint *ep);
+void dmrC_track_pseudo_death(struct dmr_C *C, struct entrypoint *ep);
+void dmrC_track_phi_uses(struct dmr_C *C, struct instruction *insn);
+
+void dmrC_init_linearizer(struct dmr_C *C);
+void dmrC_destroy_linearizer(struct dmr_C *C);
+
+#define dmrC_hashval(x) ((unsigned long)(((uintptr_t)(x))))
+
+#define REPEAT_CSE		1
+#define REPEAT_SYMBOL_CLEANUP	2
+#define REPEAT_CFG_CLEANUP	3
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* LINEARIZE_H */
+

dmr_c/src/liveness.c

@@ -0,0 +1,384 @@
+/*
+ * Register - track pseudo usage, maybe eventually try to do register
+ * allocation.
+ *
+ * Copyright (C) 2004 Linus Torvalds
+ */
+
+#include <assert.h>
+
+#include <port.h>
+#include <parse.h>
+#include <expression.h>
+#include <linearize.h>
+//#include <flow.h>
+
+static void phi_defines(struct dmr_C *C, struct instruction * phi_node, pseudo_t target,
+	void (*defines)(struct dmr_C *C, struct basic_block *, pseudo_t))
+{
+	pseudo_t phi;
+	FOR_EACH_PTR(phi_node->phi_list, phi) {
+		struct instruction *def;
+		if (phi == VOID_PSEUDO(C))
+			continue;
+		def = phi->def;
+		if (!def || !def->bb)
+			continue;
+		defines(C, def->bb, target);
+	} END_FOR_EACH_PTR(phi);
+}
+
+static void asm_liveness(struct dmr_C *C, struct basic_block *bb, struct instruction *insn,
+	void (*def)(struct dmr_C *C, struct basic_block *, pseudo_t),
+	void (*use)(struct dmr_C *C, struct basic_block *, pseudo_t))
+{
+	struct asm_constraint *entry;
+
+	FOR_EACH_PTR(insn->asm_rules->inputs, entry) {
+		use(C, bb, entry->pseudo);
+	} END_FOR_EACH_PTR(entry);
+		
+	FOR_EACH_PTR(insn->asm_rules->outputs, entry) {
+		def(C, bb, entry->pseudo);
+	} END_FOR_EACH_PTR(entry);
+}
+
+static void track_instruction_usage(struct dmr_C *C, struct basic_block *bb, struct instruction *insn,
+	void (*def)(struct dmr_C *C, struct basic_block *, pseudo_t),
+	void (*use)(struct dmr_C *C, struct basic_block *, pseudo_t))
+{
+	#define USES(x)		use(C, bb, insn->x)
+	#define DEFINES(x)	def(C, bb, insn->x)
+
+	switch (insn->opcode) {
+	case OP_RET:
+		USES(src);
+		break;
+
+	case OP_CBR: 
+	case OP_SWITCH:
+		USES(cond);
+		break;
+
+	case OP_COMPUTEDGOTO:
+		USES(target);
+		break;
+	
+	/* Binary */
+	case OP_ADD:
+	case OP_SUB:
+	case OP_MULU:
+	case OP_MULS:
+	case OP_DIVU:
+	case OP_DIVS:
+	case OP_MODU:
+	case OP_MODS:
+	case OP_SHL:
+	case OP_LSR:
+	case OP_ASR:
+	case OP_AND:
+	case OP_OR:
+	case OP_XOR:
+	case OP_AND_BOOL:
+	case OP_OR_BOOL:
+	case OP_SET_EQ:
+	case OP_SET_NE:
+	case OP_SET_LE:
+	case OP_SET_GE:
+	case OP_SET_LT:
+	case OP_SET_GT:
+	case OP_SET_B:
+	case OP_SET_A:
+	case OP_SET_BE:
+	case OP_SET_AE:
+		USES(src1); USES(src2); DEFINES(target);
+		break;
+
+	/* Uni */
+	case OP_NOT: case OP_NEG:
+		USES(src1); DEFINES(target);
+		break;
+
+	case OP_SEL:
+		USES(src1); USES(src2); USES(src3); DEFINES(target);
+		break;
+	
+	/* Memory */
+	case OP_LOAD:
+		USES(src); DEFINES(target);
+		break;
+
+	case OP_STORE:
+		USES(src); USES(target);
+		break;
+
+	case OP_SETVAL:
+		DEFINES(target);
+		break;
+
+	case OP_SYMADDR:
+		USES(symbol); DEFINES(target);
+		break;
+
+	/* Other */
+	case OP_PHI:
+		/* Phi-nodes are "backwards" nodes. Their def doesn't matter */
+		phi_defines(C, insn, insn->target, def);
+		break;
+
+	case OP_PHISOURCE:
+		/*
+		 * We don't care about the phi-source define, they get set
+		 * up and expanded by the OP_PHI
+		 */
+		USES(phi_src);
+		break;
+
+	case OP_CAST:
+	case OP_SCAST:
+	case OP_FPCAST:
+	case OP_PTRCAST:
+		USES(src); DEFINES(target);
+		break;
+
+	case OP_CALL: {
+		pseudo_t arg;
+		USES(func);
+		if (insn->target != VOID_PSEUDO(C))
+			DEFINES(target);
+		FOR_EACH_PTR(insn->arguments, arg) {
+			use(C, bb, arg);
+		} END_FOR_EACH_PTR(arg);
+		break;
+	}
+
+	case OP_SLICE:
+		USES(base); DEFINES(target);
+		break;
+
+	case OP_ASM:
+		asm_liveness(C, bb, insn, def, use);
+		break;
+
+	case OP_RANGE:
+		USES(src1); USES(src2); USES(src3);
+		break;
+
+	case OP_BADOP:
+	case OP_INVOKE:
+	case OP_UNWIND:
+	case OP_MALLOC:
+	case OP_FREE:
+	case OP_ALLOCA:
+	case OP_GET_ELEMENT_PTR:
+	case OP_VANEXT:
+	case OP_VAARG:
+	case OP_SNOP:
+	case OP_LNOP:
+	case OP_NOP:
+	case OP_CONTEXT:
+		break;
+	}
+}
+
+int dmrC_pseudo_in_list(struct pseudo_list *list, pseudo_t pseudo)
+{
+	pseudo_t old;
+	FOR_EACH_PTR(list,old) {
+		if (old == pseudo)
+			return 1;
+	} END_FOR_EACH_PTR(old);   
+	return 0;
+}
+
+static void add_pseudo_exclusive(struct dmr_C *C, struct pseudo_list **list, pseudo_t pseudo)
+{
+	if (!dmrC_pseudo_in_list(*list, pseudo)) {
+		C->L->liveness_changed = 1;
+		dmrC_add_pseudo(C, list, pseudo);
+	}
+}
+
+static inline int trackable_pseudo(pseudo_t pseudo)
+{
+	return pseudo && (pseudo->type == PSEUDO_REG || pseudo->type == PSEUDO_ARG);
+}
+
+static void insn_uses(struct dmr_C *C, struct basic_block *bb, pseudo_t pseudo)
+{
+	if (trackable_pseudo(pseudo)) {
+		struct instruction *def = pseudo->def;
+		if (pseudo->type != PSEUDO_REG || def->bb != bb || def->opcode == OP_PHI)
+			add_pseudo_exclusive(C, &bb->needs, pseudo);
+	}
+}
+
+static void insn_defines(struct dmr_C *C, struct basic_block *bb, pseudo_t pseudo)
+{
+	assert(trackable_pseudo(pseudo));
+	dmrC_add_pseudo(C, &bb->defines, pseudo);
+}
+
+static void track_bb_liveness(struct dmr_C *C, struct basic_block *bb)
+{
+	pseudo_t needs;
+
+	FOR_EACH_PTR(bb->needs, needs) {
+		struct basic_block *parent;
+		FOR_EACH_PTR(bb->parents, parent) {
+			if (!dmrC_pseudo_in_list(parent->defines, needs)) {
+				add_pseudo_exclusive(C, &parent->needs, needs);
+			}
+		} END_FOR_EACH_PTR(parent);
+	} END_FOR_EACH_PTR(needs);
+}
+
+/*
+ * We need to clear the liveness information if we 
+ * are going to re-run it.
+ */
+void dmrC_clear_liveness(struct entrypoint *ep)
+{
+	struct basic_block *bb;
+
+	FOR_EACH_PTR(ep->bbs, bb) {
+		ptrlist_remove_all((struct ptr_list **)&bb->needs);
+		ptrlist_remove_all((struct ptr_list **)&bb->defines);
+	} END_FOR_EACH_PTR(bb);
+}
+
+/*
+ * Track inter-bb pseudo liveness. The intra-bb case
+ * is purely local information.
+ */
+void dmrC_track_pseudo_liveness(struct dmr_C *C, struct entrypoint *ep)
+{
+	struct basic_block *bb;
+
+	/* Add all the bb pseudo usage */
+	FOR_EACH_PTR(ep->bbs, bb) {
+		struct instruction *insn;
+		FOR_EACH_PTR(bb->insns, insn) {
+			if (!insn->bb)
+				continue;
+			assert(insn->bb == bb);
+			track_instruction_usage(C, bb, insn, insn_defines, insn_uses);
+		} END_FOR_EACH_PTR(insn);
+	} END_FOR_EACH_PTR(bb);
+
+	/* Calculate liveness.. */
+	do {
+		C->L->liveness_changed = 0;
+		FOR_EACH_PTR_REVERSE(ep->bbs, bb) {
+			track_bb_liveness(C, bb);
+		} END_FOR_EACH_PTR_REVERSE(bb);
+	} while (C->L->liveness_changed);
+
+	/* Remove the pseudos from the "defines" list that are used internally */
+	FOR_EACH_PTR(ep->bbs, bb) {
+		pseudo_t def;
+		FOR_EACH_PTR(bb->defines, def) {
+			struct basic_block *child;
+			FOR_EACH_PTR(bb->children, child) {
+				if (dmrC_pseudo_in_list(child->needs, def))
+					goto is_used;
+			} END_FOR_EACH_PTR(child);
+			DELETE_CURRENT_PTR(def);
+is_used:
+		;
+		} END_FOR_EACH_PTR(def);
+		ptrlist_pack((struct ptr_list **)&bb->defines);
+	} END_FOR_EACH_PTR(bb);
+}
+
+static void merge_pseudo_list(struct dmr_C *C, struct pseudo_list *src, struct pseudo_list **dest)
+{
+	pseudo_t pseudo;
+	FOR_EACH_PTR(src, pseudo) {
+		add_pseudo_exclusive(C, dest, pseudo);
+	} END_FOR_EACH_PTR(pseudo);
+}
+
+void dmrC_track_phi_uses(struct dmr_C *C, struct instruction *insn)
+{
+	pseudo_t phi;
+	FOR_EACH_PTR(insn->phi_list, phi) {
+		struct instruction *def;
+		if (phi == VOID_PSEUDO(C) || !phi->def)
+			continue;
+		def = phi->def;
+		assert(def->opcode == OP_PHISOURCE);
+        dmrC_add_instruction(C, &def->phi_users, insn);
+	} END_FOR_EACH_PTR(phi);
+}
+
+static void track_bb_phi_uses(struct dmr_C *C, struct basic_block *bb)
+{
+	struct instruction *insn;
+	FOR_EACH_PTR(bb->insns, insn) {
+		if (insn->bb && insn->opcode == OP_PHI)
+			dmrC_track_phi_uses(C, insn);
+	} END_FOR_EACH_PTR(insn);
+}
+
+static void death_def(struct dmr_C *C, struct basic_block *bb, pseudo_t pseudo)
+{
+	(void) C;
+	(void) bb;
+	(void) pseudo;
+}
+
+static void death_use(struct dmr_C *C, struct basic_block *bb, pseudo_t pseudo)
+{
+	(void) C;
+	(void) bb;
+	if (trackable_pseudo(pseudo) && !dmrC_pseudo_in_list(*C->L->live_list, pseudo)) {
+		dmrC_add_pseudo(C, &C->L->dead_list, pseudo);
+		dmrC_add_pseudo(C, C->L->live_list, pseudo);
+	}
+}
+
+static void track_pseudo_death_bb(struct dmr_C *C, struct basic_block *bb)
+{
+	struct pseudo_list *live = NULL;
+	struct basic_block *child;
+	struct instruction *insn;
+
+	FOR_EACH_PTR(bb->children, child) {
+		merge_pseudo_list(C, child->needs, &live);
+	} END_FOR_EACH_PTR(child);
+
+	C->L->live_list = &live;
+	FOR_EACH_PTR_REVERSE(bb->insns, insn) {
+		if (!insn->bb)
+			continue;
+
+		C->L->dead_list = NULL;
+		track_instruction_usage(C, bb, insn, death_def, death_use);
+		if (C->L->dead_list) {
+			pseudo_t dead;
+			FOR_EACH_PTR(C->L->dead_list, dead) {
+				struct instruction *deathnote = dmrC_allocator_allocate(&C->L->instruction_allocator, 0);
+				deathnote->bb = bb;
+				deathnote->opcode = OP_DEATHNOTE;
+				deathnote->target = dead;
+				INSERT_CURRENT(deathnote, insn);
+			} END_FOR_EACH_PTR(dead);
+			ptrlist_remove_all((struct ptr_list **)&C->L->dead_list);
+		}
+	} END_FOR_EACH_PTR_REVERSE(insn);
+	ptrlist_remove_all((struct ptr_list **)&live);
+}
+
+void dmrC_track_pseudo_death(struct dmr_C *C, struct entrypoint *ep)
+{
+	struct basic_block *bb;
+
+	FOR_EACH_PTR(ep->bbs, bb) {
+		track_bb_phi_uses(C, bb);
+	} END_FOR_EACH_PTR(bb);
+
+	FOR_EACH_PTR(ep->bbs, bb) {
+		track_pseudo_death_bb(C, bb);
+	} END_FOR_EACH_PTR(bb);
+}

dmr_c/src/memops.c

@@ -0,0 +1,196 @@
+/*
+ * memops - try to combine memory ops.
+ *
+ * Copyright (C) 2004 Linus Torvalds
+ */
+
+#include <string.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <assert.h>
+
+#include <port.h>
+#include <parse.h>
+#include <expression.h>
+#include <linearize.h>
+#include <flow.h>
+
+static int find_dominating_parents(struct dmr_C *C, pseudo_t pseudo, struct instruction *insn,
+	struct basic_block *bb, unsigned long generation, struct pseudo_list **dominators,
+	int local)
+{
+	struct basic_block *parent;
+
+	FOR_EACH_PTR(bb->parents, parent) {
+		struct instruction *one;
+		struct instruction *br;
+		pseudo_t phi;
+
+		FOR_EACH_PTR_REVERSE(parent->insns, one) {
+			int dominance;
+			if (!one->bb)
+				continue;
+			if (one == insn)
+				goto no_dominance;
+			dominance = dmrC_dominates(C, pseudo, insn, one, local);
+			if (dominance < 0) {
+				if (one->opcode == OP_LOAD)
+					continue;
+				return 0;
+			}
+			if (!dominance)
+				continue;
+			goto found_dominator;
+		} END_FOR_EACH_PTR_REVERSE(one);
+no_dominance:
+		if (parent->generation == generation)
+			continue;
+		parent->generation = generation;
+
+		if (!find_dominating_parents(C, pseudo, insn, parent, generation, dominators, local))
+			return 0;
+		continue;
+
+found_dominator:
+		br = dmrC_delete_last_instruction(&parent->insns);
+		phi = dmrC_alloc_phi(C, parent, one->target, one->type);
+		phi->ident = phi->ident ? phi->ident: one->target->ident;
+		dmrC_add_instruction(C, &parent->insns, br);
+		dmrC_use_pseudo(C, insn, phi, dmrC_add_pseudo(C, dominators, phi));
+	} END_FOR_EACH_PTR(parent);
+	return 1;
+}		
+
+static int address_taken(pseudo_t pseudo)
+{
+	struct pseudo_user *pu;
+	FOR_EACH_PTR(pseudo->users, pu) {
+		struct instruction *insn = pu->insn;
+		if (insn->bb && (insn->opcode != OP_LOAD && insn->opcode != OP_STORE))
+			return 1;
+	} END_FOR_EACH_PTR(pu);
+	return 0;
+}
+
+static int local_pseudo(pseudo_t pseudo)
+{
+	return pseudo->type == PSEUDO_SYM
+		&& !(pseudo->sym->ctype.modifiers & (MOD_STATIC | MOD_NONLOCAL))
+		&& !address_taken(pseudo);
+}
+
+static void simplify_loads(struct dmr_C *C, struct basic_block *bb)
+{
+	struct instruction *insn;
+
+	FOR_EACH_PTR_REVERSE(bb->insns, insn) {
+		if (!insn->bb)
+			continue;
+		if (insn->opcode == OP_LOAD) {
+			struct instruction *dom;
+			pseudo_t pseudo = insn->src;
+			int local = local_pseudo(pseudo);
+			struct pseudo_list *dominators;
+			unsigned long generation;
+
+			/* Check for illegal offsets.. */
+			dmrC_check_access(C, insn);
+			
+			if (insn->type->ctype.modifiers & MOD_VOLATILE)
+				continue;
+			RECURSE_PTR_REVERSE(insn, dom) {
+				int dominance;
+				if (!dom->bb)
+					continue;
+				dominance = dmrC_dominates(C, pseudo, insn, dom, local);
+				if (dominance) {
+					/* possible partial dominance? */
+					if (dominance < 0)  {
+						if (dom->opcode == OP_LOAD)
+							continue;
+						goto next_load;
+					}
+					/* Yeehaa! Found one! */
+					dmrC_convert_load_instruction(C, insn, dom->target);
+					goto next_load;
+				}
+			} END_FOR_EACH_PTR_REVERSE(dom);
+			/* OK, go find the parents */
+			generation = ++C->L->bb_generation;
+			bb->generation = generation;
+			dominators = NULL;
+			if (find_dominating_parents(C, pseudo, insn, bb, generation, &dominators, local)) {
+				/* This happens with initial assignments to structures etc.. */
+				if (!dominators) {
+					if (local) {
+						assert(pseudo->type != PSEUDO_ARG);
+						dmrC_convert_load_instruction(C, insn, dmrC_value_pseudo(C, insn->type, 0));
+					}
+					goto next_load;
+				}
+				dmrC_rewrite_load_instruction(C, insn, dominators);
+			}
+		}
+next_load:
+		/* Do the next one */;
+	} END_FOR_EACH_PTR_REVERSE(insn);
+}
+
+static void kill_store(struct dmr_C *C, struct instruction *insn)
+{
+	if (insn) {
+		insn->bb = NULL;
+		insn->opcode = OP_SNOP;
+		dmrC_kill_use(C, &insn->target);
+	}
+}
+
+static void kill_dominated_stores(struct dmr_C *C, struct basic_block *bb)
+{
+	struct instruction *insn;
+
+	FOR_EACH_PTR_REVERSE(bb->insns, insn) {
+		if (!insn->bb)
+			continue;
+		if (insn->opcode == OP_STORE) {
+			struct instruction *dom;
+			pseudo_t pseudo = insn->src;
+			int local = local_pseudo(pseudo);
+
+			RECURSE_PTR_REVERSE(insn, dom) {
+				int dominance;
+				if (!dom->bb)
+					continue;
+				dominance = dmrC_dominates(C, pseudo, insn, dom, local);
+				if (dominance) {
+					/* possible partial dominance? */
+					if (dominance < 0)
+						goto next_store;
+					if (dom->opcode == OP_LOAD)
+						goto next_store;
+					/* Yeehaa! Found one! */
+					kill_store(C, dom);
+				}
+			} END_FOR_EACH_PTR_REVERSE(dom);
+
+			/* OK, we should check the parents now */
+		}
+next_store:
+		/* Do the next one */;
+	} END_FOR_EACH_PTR_REVERSE(insn);
+}
+
+void dmrC_simplify_memops(struct dmr_C *C, struct entrypoint *ep)
+{
+	struct basic_block *bb;
+
+	FOR_EACH_PTR_REVERSE(ep->bbs, bb) {
+		simplify_loads(C, bb);
+	} END_FOR_EACH_PTR_REVERSE(bb);
+
+	FOR_EACH_PTR_REVERSE(ep->bbs, bb) {
+		kill_dominated_stores(C, bb);
+	} END_FOR_EACH_PTR_REVERSE(bb);
+}

dmr_c/src/parse.h

@@ -0,0 +1,185 @@
+#ifndef DMR_C_PARSE_H
+#define DMR_C_PARSE_H
+/*
+* Basic parsing data structures. Statements and symbols.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <lib.h>
+#include <symbol.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+enum statement_type {
+	STMT_NONE,
+	STMT_DECLARATION,
+	STMT_EXPRESSION,
+	STMT_COMPOUND,
+	STMT_IF,
+	STMT_RETURN,
+	STMT_CASE,
+	STMT_SWITCH,
+	STMT_ITERATOR,
+	STMT_LABEL,
+	STMT_GOTO,
+	STMT_ASM,
+	STMT_CONTEXT,
+	STMT_RANGE,
+};
+
+DECLARE_PTR_LIST(statement_list, struct statement);
+
+struct statement {
+	enum statement_type type;
+	struct position pos;
+	union {
+		struct /* declaration */ {
+			struct symbol_list *declaration;
+		};
+		struct {
+			struct expression *expression;
+			struct expression *context;
+		};
+		struct /* return_statement */ {
+			struct expression *ret_value;
+			struct symbol *ret_target;
+		};
+		struct /* if_statement */ {
+			struct expression *if_conditional;
+			struct statement *if_true;
+			struct statement *if_false;
+		};
+		struct /* compound_struct */ {
+			struct statement_list *stmts;
+			struct symbol *ret;
+			struct symbol *inline_fn;
+			struct statement *args;
+		};
+		struct /* labeled_struct */ {
+			struct symbol *label_identifier;
+			struct statement *label_statement;
+		};
+		struct /* case_struct */ {
+			struct expression *case_expression;
+			struct expression *case_to;
+			struct statement *case_statement;
+			struct symbol *case_label;
+		};
+		struct /* switch_struct */ {
+			struct expression *switch_expression;
+			struct statement *switch_statement;
+			struct symbol *switch_break, *switch_case;
+		};
+		struct /* iterator_struct */ {
+			struct symbol *iterator_break;
+			struct symbol *iterator_continue;
+			struct symbol_list *iterator_syms;
+			struct statement *iterator_pre_statement;
+			struct expression *iterator_pre_condition;
+
+			struct statement  *iterator_statement;
+
+			struct statement  *iterator_post_statement;
+			struct expression *iterator_post_condition;
+		};
+		struct /* goto_struct */ {
+			struct symbol *goto_label;
+
+			/* computed gotos have these: */
+			struct expression *goto_expression;
+			struct symbol_list *target_list;
+		};
+		struct /* asm */ {
+			struct expression *asm_string;
+			struct expression_list *asm_outputs;
+			struct expression_list *asm_inputs;
+			struct expression_list *asm_clobbers;
+			struct symbol_list *asm_labels;
+		};
+		struct /* range */ {
+			struct expression *range_expression;
+			struct expression *range_low;
+			struct expression *range_high;
+		};
+	};
+};
+
+struct parse_state_t {
+	struct symbol_list **function_symbol_list;
+	struct symbol_list *function_computed_target_list;
+	struct statement_list *function_computed_goto_list;
+	struct symbol * int_types[4];
+	struct symbol * signed_types[5];
+	struct symbol * unsigned_types[5];
+	struct symbol * real_types[3];
+	struct symbol * char_types[3];
+	struct symbol ** types[7];
+};
+
+extern void dmrC_init_parser(struct dmr_C *C, int stream);
+void dmrC_destroy_parser(struct dmr_C *C);
+
+extern struct token *dmrC_parse_expression(struct dmr_C *C, struct token *, struct expression **);
+extern struct symbol *dmrC_label_symbol(struct dmr_C *C, struct token *token);
+
+extern int dmrC_show_statement(struct dmr_C *C, struct statement *);
+extern int dmrC_show_expression(struct dmr_C *C, struct expression *);
+typedef void(*validate_decl_t)(struct dmr_C *C, struct symbol *decl);
+extern struct token *dmrC_external_declaration(struct dmr_C *C, struct token *token, struct symbol_list **symbol_list, validate_decl_t);
+
+extern struct symbol *dmrC_ctype_integer(struct dmr_C *C, int size, int want_unsigned);
+
+extern void dmrC_copy_statement(struct dmr_C *C, struct statement *src, struct statement *dst);
+extern int dmrC_inline_function(struct dmr_C *C, struct expression *expr, struct symbol *sym);
+extern void dmrC_uninline(struct dmr_C *C, struct symbol *sym);
+
+static inline void dmrC_add_statement(struct dmr_C *C, struct statement_list **list, struct statement *stmt)
+{
+	ptrlist_add((struct ptr_list **)list, stmt, &C->ptrlist_allocator);
+}
+
+static inline void dmrC_add_expression(struct dmr_C *C, struct expression_list **list, struct expression *expr)
+{
+	ptrlist_add((struct ptr_list **)list, expr, &C->ptrlist_allocator);
+}
+
+static inline int dmrC_expression_list_size(struct expression_list *list)
+{
+	return ptrlist_size((struct ptr_list *)list);
+}
+
+extern int dmrC_test_parse();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* PARSE_H */

dmr_c/src/port.h

@@ -0,0 +1,59 @@
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#ifndef DMR_C_PORT_H
+#define DMR_C_PORT_H
+
+#ifndef _WIN32
+#include <unistd.h>
+#include <alloca.h>
+#else
+#include <io.h>
+#include <malloc.h>
+#endif
+
+#if defined(_WIN32) && defined(_MSC_VER)
+#include <stdlib.h>
+#ifndef __cplusplus
+#define inline __inline
+#endif
+#define __alignof__ __alignof
+#define __builtin_bswap16 _byteswap_ushort
+#define __builtin_bswap32 _byteswap_ulong
+#define __builtin_bswap64 _byteswap_uint64
+#endif
+
+#ifdef __GNUC__
+#define FORMAT_ATTR(pos) __attribute__((__format__(__printf__, pos, pos + 1)))
+#define NORETURN_ATTR __attribute__((__noreturn__))
+#define SENTINEL_ATTR __attribute__((__sentinel__))
+#else
+#define FORMAT_ATTR(pos)
+#define NORETURN_ATTR
+#define SENTINEL_ATTR
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void *dmrC_blob_alloc(size_t size);
+void dmrC_blob_free(void *addr, size_t size);
+long double dmrC_string_to_ld(const char *nptr, char **endptr);
+
+#include <stdint.h>
+
+#ifdef ASMJIT_STATIC
+struct backend_data { uint64_t x[2]; };
+#define DMRC_BACKEND_TYPE struct backend_data
+#else
+#define DMRC_BACKEND_TYPE void *
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

dmr_c/src/ptrlist.c

@@ -0,0 +1,977 @@
+/*
+* ptrlist.c
+*
+* Pointer ptrlist_t manipulation
+*
+* (C) Copyright Linus Torvalds 2003-2005
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#define PARANOIA 1
+
+#include <ptrlist.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/* The ptr list */
+
+/* For testing we change this */
+static int N_ = LIST_NODE_NR;
+
+void ptrlist_split_node(struct ptr_list *head)
+{
+	int old = head->nr_, nr = old / 2;
+	struct allocator *alloc = head->allocator_;
+	assert(alloc);
+	struct ptr_list *newlist =
+	    (struct ptr_list *)dmrC_allocator_allocate(alloc, 0);
+	struct ptr_list *next = head->next_;
+	newlist->allocator_ = alloc;
+
+	old -= nr;
+	head->nr_ = old;
+	newlist->next_ = next;
+	next->prev_ = newlist;
+	newlist->prev_ = head;
+	head->next_ = newlist;
+	newlist->nr_ = nr;
+	memcpy(newlist->list_, head->list_ + old, nr * sizeof(void *));
+	memset(head->list_ + old, 0xf0, nr * sizeof(void *));
+}
+
+struct ptr_list_iter ptrlist_forward_iterator(struct ptr_list *head)
+{
+	struct ptr_list_iter iter;
+	iter.__head = iter.__list = head;
+	iter.__nr = -1;
+	return iter;
+}
+
+// Reverse iterator has to start from previous node not previous entry
+// in the given head
+struct ptr_list_iter ptrlist_reverse_iterator(struct ptr_list *head)
+{
+	struct ptr_list_iter iter;
+	iter.__head = iter.__list = head ? head->prev_ : NULL;
+	iter.__nr = iter.__head ? iter.__head->nr_ : 0;
+	return iter;
+}
+
+void *ptrlist_iter_next(struct ptr_list_iter *self)
+{
+	if (self->__head == NULL)
+		return NULL;
+	self->__nr++;
+Lretry:
+	if (self->__nr < self->__list->nr_) {
+		void *ptr = self->__list->list_[self->__nr];
+		if (self->__list->rm_ && !ptr) {
+			self->__nr++;
+			goto Lretry;
+		}
+		return ptr;
+	} else if (self->__list->next_ != self->__head) {
+		self->__list = self->__list->next_;
+		self->__nr = 0;
+		goto Lretry;
+	}
+	return NULL;
+}
+
+void *ptrlist_nth_entry(struct ptr_list *list, unsigned int idx)
+{
+	struct ptr_list *head = list;
+	if (!head)
+		return NULL;
+	do {
+		unsigned int nr = list->nr_;
+		if (idx < nr)
+			return list->list_[idx];
+		else
+			idx -= nr;
+	} while ((list = list->next_) != head);
+	return NULL;
+}
+
+void *ptrlist_iter_prev(struct ptr_list_iter *self)
+{
+	if (self->__head == NULL)
+		return NULL;
+	self->__nr--;
+Lretry:
+	if (self->__nr >= 0 && self->__nr < self->__list->nr_) {
+		void *ptr = self->__list->list_[self->__nr];
+		if (self->__list->rm_ && !ptr) {
+			self->__nr--;
+			goto Lretry;
+		}
+		return ptr;
+	} else if (self->__list->prev_ != self->__head) {
+		self->__list = self->__list->prev_;
+		self->__nr = self->__list->nr_ - 1;
+		goto Lretry;
+	}
+	return NULL;
+}
+
+void ptrlist_iter_split_current(struct ptr_list_iter *self)
+{
+	if (self->__list->nr_ == N_) {
+		/* full so split */
+		ptrlist_split_node(self->__list);
+		if (self->__nr >= self->__list->nr_) {
+			self->__nr -= self->__list->nr_;
+			self->__list = self->__list->next_;
+		}
+	}
+}
+
+void ptrlist_iter_insert(struct ptr_list_iter *self, void *newitem)
+{
+	assert(self->__nr >= 0);
+	ptrlist_iter_split_current(self);
+	void **__this = self->__list->list_ + self->__nr;
+	void **__last = self->__list->list_ + self->__list->nr_ - 1;
+	while (__last >= __this) {
+		__last[1] = __last[0];
+		__last--;
+	}
+	*__this = newitem;
+	self->__list->nr_++;
+}
+
+void ptrlist_iter_remove(struct ptr_list_iter *self)
+{
+	assert(self->__nr >= 0);
+	void **__this = self->__list->list_ + self->__nr;
+	void **__last = self->__list->list_ + self->__list->nr_ - 1;
+	while (__this < __last) {
+		__this[0] = __this[1];
+		__this++;
+	}
+	*__this = (void *)((uintptr_t)0xf0f0f0f0);
+	self->__list->nr_--;
+	self->__nr--;
+}
+
+void ptrlist_iter_set(struct ptr_list_iter *self, void *ptr)
+{
+	assert(self->__list && self->__nr >= 0 &&
+	       self->__nr < self->__list->nr_);
+	self->__list->list_[self->__nr] = ptr;
+}
+
+void ptrlist_iter_mark_deleted(struct ptr_list_iter *self)
+{
+	ptrlist_iter_set(self, NULL);
+	self->__list->rm_++;
+}
+
+int ptrlist_size(const struct ptr_list *head) {
+	int nr = 0;
+	if (head) {
+		const struct ptr_list *list = head;
+		do {
+			nr += list->nr_ - list->rm_;
+		} while ((list = list->next_) != head);
+	}
+	return nr;
+}
+
+void **ptrlist_add(struct ptr_list **listp, void *ptr, struct allocator *alloc)
+{
+	struct ptr_list *list = *listp;
+	struct ptr_list *last = NULL;
+	void **ret;
+	int nr;
+
+	if (!list || (nr = (last = list->prev_)->nr_) >= N_) {
+		struct ptr_list *newlist =
+		    (struct ptr_list *)dmrC_allocator_allocate(alloc, 0);
+		newlist->allocator_ = alloc;
+		if (!list) {
+			newlist->next_ = newlist;
+			newlist->prev_ = newlist;
+			*listp = newlist;
+		} else {
+			newlist->prev_ = last;
+			newlist->next_ = list;
+			list->prev_ = newlist;
+			last->next_ = newlist;
+		}
+		last = newlist;
+		nr = 0;
+	}
+	ret = last->list_ + nr;
+	*ret = ptr;
+	nr++;
+	last->nr_ = nr;
+	return ret;
+}
+
+void *ptrlist_first(struct ptr_list *list)
+{
+	if (!list)
+		return NULL;
+	return list->list_[0];
+}
+
+void *ptrlist_last(struct ptr_list *list)
+{
+	if (!list)
+		return NULL;
+	list = list->prev_;
+	return list->list_[list->nr_ - 1];
+}
+
+/*
+* Linearize the entries of a list up to a total of 'max',
+* and return the nr of entries linearized.
+*
+* The array to linearize into (second argument) should really
+* be "void *x[]", but we want to let people fill in any kind
+* of pointer array, so let's just call it "void **".
+*/
+int ptrlist_linearize(struct ptr_list *head, void **arr, int max) {
+	int nr = 0;
+	if (head && max > 0) {
+		struct ptr_list *list = head;
+
+		do {
+			int i = list->nr_;
+			if (i > max)
+				i = max;
+			memcpy(arr, list->list_, i * sizeof(void *));
+			arr += i;
+			nr += i;
+			max -= i;
+			if (!max)
+				break;
+		} while ((list = list->next_) != head);
+	}
+	return nr;
+}
+
+/*
+* When we've walked the list and deleted entries,
+* we may need to re-pack it so that we don't have
+* any empty blocks left (empty blocks upset the
+* walking code
+*/
+void ptrlist_pack(struct ptr_list **listp)
+{
+	struct ptr_list *head = *listp;
+
+	if (head) {
+		struct ptr_list *entry = head;
+		do {
+			struct ptr_list *next;
+		restart:
+			next = entry->next_;
+			if (!entry->nr_) {
+				struct ptr_list *prev;
+				if (next == entry) {
+					dmrC_allocator_free(entry->allocator_, entry);
+					*listp = NULL;
+					return;
+				}
+				prev = entry->prev_;
+				prev->next_ = next;
+				next->prev_ = prev;
+				dmrC_allocator_free(entry->allocator_, entry);
+				if (entry == head) {
+					*listp = next;
+					head = next;
+					entry = next;
+					goto restart;
+				}
+			}
+			entry = next;
+		} while (entry != head);
+	}
+}
+
+void ptrlist_remove_all(struct ptr_list **listp) {
+	struct ptr_list *tmp, *list = *listp;
+	if (!list)
+		return;
+	list->prev_->next_ = NULL;
+	while (list) {
+		tmp = list;
+		list = list->next_;
+		dmrC_allocator_free(tmp->allocator_, tmp);
+	}
+	*listp = NULL;
+}
+
+
+int ptrlist_remove(struct ptr_list **self, void *entry, int count) {
+  struct ptr_list_iter iter = ptrlist_forward_iterator(*self);
+  for (void *ptr = ptrlist_iter_next(&iter); ptr != NULL;
+       ptr = ptrlist_iter_next(&iter)) {
+    if (ptr == entry) {
+      ptrlist_iter_remove(&iter);
+      if (!--count)
+        goto out;
+    }
+  }
+  assert(count <= 0);
+out:
+  ptrlist_pack(self);
+  return count;
+}
+
+int ptrlist_replace(struct ptr_list **self, void *old_ptr, void *new_ptr,
+                    int count) {
+  struct ptr_list_iter iter = ptrlist_forward_iterator(*self);
+  for (void *ptr = ptrlist_iter_next(&iter); ptr != NULL;
+       ptr = ptrlist_iter_next(&iter)) {
+    if (ptr == old_ptr) {
+      ptrlist_iter_set(&iter, new_ptr);
+      if (!--count)
+        goto out;
+    }
+  }
+  assert(count <= 0);
+out:
+  return count;
+}
+
+/* This removes the last entry, but doesn't pack the ptr list */
+void *ptrlist_undo_last(struct ptr_list **head)
+{
+	struct ptr_list *last, *first = *head;
+
+	if (!first)
+		return NULL;
+	last = first;
+	do {
+		last = last->prev_;
+		if (last->nr_) {
+			void *ptr;
+			int nr = --last->nr_;
+			ptr = last->list_[nr];
+			last->list_[nr] = (void *)((intptr_t)0xf1f1f1f1);
+			return ptr;
+		}
+	} while (last != first);
+	return NULL;
+}
+
+
+void *ptrlist_delete_last(struct ptr_list **head)
+{
+	void *ptr = NULL;
+	struct ptr_list *last, *first = *head;
+
+	if (!first)
+		return NULL;
+	last = first->prev_;
+	if (last->nr_)
+		ptr = last->list_[--last->nr_];
+	if (last->nr_ <= 0) {
+		first->prev_ = last->prev_;
+		last->prev_->next_ = first;
+		if (last == first)
+			*head = NULL;
+		dmrC_allocator_free(last->allocator_, last);
+	}
+	return ptr;
+}
+
+void ptrlist_concat(struct ptr_list *a, struct ptr_list **b) {
+  struct allocator *alloc = NULL;
+  struct ptr_list_iter iter = ptrlist_forward_iterator(a);
+  if (a)
+	  alloc = a->allocator_;
+  else if (*b)
+	  alloc = (*b)->allocator_;
+  else
+	  return;
+  for (void *ptr = ptrlist_iter_next(&iter); ptr != NULL;
+       ptr = ptrlist_iter_next(&iter)) {
+    ptrlist_add(b, ptr, alloc);
+  }
+}
+
+/*
+* sort_list: a stable sort for lists.
+*
+* Time complexity: O(n*log n)
+*   [assuming limited zero-element fragments]
+*
+* Space complexity: O(1).
+*
+* Stable: yes.
+*/
+
+static void array_sort(void **ptr, int nr, void *userdata,
+                       int (*cmp)(void *, const void *, const void *)) {
+  int i;
+  for (i = 1; i < nr; i++) {
+    void *p = ptr[i];
+    if (cmp(userdata, ptr[i - 1], p) > 0) {
+      int j = i;
+      do {
+        ptr[j] = ptr[j - 1];
+        if (!--j)
+          break;
+      } while (cmp(userdata, ptr[j - 1], p) > 0);
+      ptr[j] = p;
+    }
+  }
+}
+
+static void verify_sorted(struct ptr_list *l, int n, void *userdata,
+                          int (*cmp)(void *, const void *, const void *)) {
+  int i = 0;
+  const void *a;
+  struct ptr_list *head = l;
+
+  while (l->nr_ == 0) {
+    l = l->next_;
+    if (--n == 0)
+      return;
+    assert(l != head);
+  }
+
+  a = l->list_[0];
+  while (n > 0) {
+    const void *b;
+    if (++i >= l->nr_) {
+      i = 0;
+      l = l->next_;
+      n--;
+      assert(l != head || n == 0);
+      continue;
+    }
+    b = l->list_[i];
+    assert(cmp(userdata, a, b) <= 0);
+    a = b;
+  }
+}
+
+static void flush_to(struct ptr_list *b, void **buffer, int *nbuf) {
+  int nr = b->nr_;
+  assert(*nbuf >= nr);
+  memcpy(b->list_, buffer, nr * sizeof(void *));
+  *nbuf = *nbuf - nr;
+  memmove(buffer, buffer + nr, *nbuf * sizeof(void *));
+}
+
+static void dump_to(struct ptr_list *b, void **buffer, int nbuf) {
+  assert(nbuf <= b->nr_);
+  memcpy(b->list_, buffer, nbuf * sizeof(void *));
+}
+
+// Merge two already-sorted sequences of blocks:
+//   (b1_1, ..., b1_n)  and  (b2_1, ..., b2_m)
+// Since we may be moving blocks around, we return the new head
+// of the merged list.
+static struct ptr_list *
+merge_block_seqs(struct ptr_list *b1, int n, struct ptr_list *b2,
+                 int m, void *userdata, int (*cmp)(void *, const void *, const void *)) {
+  int i1 = 0, i2 = 0;
+  void *buffer[2 * LIST_NODE_NR];
+  int nbuf = 0;
+  struct ptr_list *newhead = b1;
+
+  // printf ("Merging %d blocks at %p with %d blocks at %p\n", n, b1, m, b2);
+
+  // Skip empty blocks in b2.
+  while (b2->nr_ == 0) {
+    // BEEN_THERE('F');
+    b2 = b2->next_;
+    if (--m == 0) {
+      // BEEN_THERE('G');
+      return newhead;
+    }
+  }
+
+  // Do a quick skip in case entire blocks from b1 are
+  // already less than smallest element in b2.
+  while (b1->nr_ == 0 || 
+  		cmp(userdata, PTR_ENTRY(b1, b1->nr_ - 1), PTR_ENTRY(b2,0)) < 0) {
+    // printf ("Skipping whole block.\n");
+    // BEEN_THERE('H');
+    b1 = b1->next_;
+    if (--n == 0) {
+      // BEEN_THERE('I');
+      return newhead;
+    }
+  }
+
+  while (1) {
+    void *d1 = PTR_ENTRY(b1,i1);
+    void *d2 = PTR_ENTRY(b2,i2);
+
+    assert(i1 >= 0 && i1 < b1->nr_);
+    assert(i2 >= 0 && i2 < b2->nr_);
+    assert(b1 != b2);
+    assert(n > 0);
+    assert(m > 0);
+
+    if (cmp(userdata, d1, d2) <= 0) {
+      // BEEN_THERE('J');
+      buffer[nbuf++] = d1;
+      // Element from b1 is smaller
+      if (++i1 >= b1->nr_) {
+        // BEEN_THERE('L');
+        flush_to(b1, buffer, &nbuf);
+        do {
+          b1 = b1->next_;
+          if (--n == 0) {
+            // BEEN_THERE('O');
+            while (b1 != b2) {
+              // BEEN_THERE('P');
+              flush_to(b1, buffer, &nbuf);
+              b1 = b1->next_;
+            }
+            assert(nbuf == i2);
+            dump_to(b2, buffer, nbuf);
+            return newhead;
+          }
+        } while (b1->nr_ == 0);
+        i1 = 0;
+      }
+    } else {
+      // BEEN_THERE('K');
+      // Element from b2 is smaller
+      buffer[nbuf++] = d2;
+      if (++i2 >= b2->nr_) {
+        struct ptr_list *l = b2;
+        // BEEN_THERE('M');
+        // OK, we finished with b2.  Pull it out
+        // and plug it in before b1.
+
+        b2 = b2->next_;
+        b2->prev_ = l->prev_;
+        b2->prev_->next_ = b2;
+        l->next_ = b1;
+        l->prev_ = b1->prev_;
+        l->next_->prev_ = l;
+        l->prev_->next_ = l;
+
+        if (b1 == newhead) {
+          // BEEN_THERE('N');
+          newhead = l;
+        }
+
+        flush_to(l, buffer, &nbuf);
+        b2 = b2->prev_;
+        do {
+          b2 = b2->next_;
+          if (--m == 0) {
+            // BEEN_THERE('Q');
+            assert(nbuf == i1);
+            dump_to(b1, buffer, nbuf);
+            return newhead;
+          }
+        } while (b2->nr_ == 0);
+        i2 = 0;
+      }
+    }
+  }
+}
+
+
+
+void ptrlist_sort(struct ptr_list **plist, void *userdata,
+                  int (*cmp)(void *, const void *, const void *)) {
+  struct ptr_list *head = *plist, *list = head;
+  int blocks = 1;
+
+  assert(N_ == LIST_NODE_NR);
+  if (!head)
+    return;
+
+  // Sort all the sub-lists
+  do {
+    array_sort(list->list_, list->nr_, userdata, cmp);
+#ifdef PARANOIA
+    verify_sorted(list, 1, userdata, cmp);
+#endif
+    list = list->next_;
+  } while (list != head);
+
+  // Merge the damn things together
+  while (1) {
+    struct ptr_list *block1 = head;
+
+    do {
+      struct ptr_list *block2 = block1;
+      struct ptr_list *next, *newhead;
+      int i;
+
+      for (i = 0; i < blocks; i++) {
+        block2 = block2->next_;
+        if (block2 == head) {
+          if (block1 == head) {
+            // BEEN_THERE('A');
+            *plist = head;
+            return;
+          }
+          // BEEN_THERE('B');
+          goto next_pass;
+        }
+      }
+
+      next = block2;
+      for (i = 0; i < blocks;) {
+        next = next->next_;
+        i++;
+        if (next == head) {
+          // BEEN_THERE('C');
+          break;
+        }
+        // BEEN_THERE('D');
+      }
+
+      newhead = merge_block_seqs(block1, blocks, block2, i, userdata, cmp);
+#ifdef PARANOIA
+      verify_sorted(newhead, blocks + i, userdata, cmp);
+#endif
+      if (block1 == head) {
+        // BEEN_THERE('E');
+        head = newhead;
+      }
+      block1 = next;
+    } while (block1 != head);
+  next_pass:
+    blocks <<= 1;
+  }
+}
+
+static int int_cmp(void *ud, const void *_a, const void *_b) {
+  (void) ud;
+  const int *a = (const int *)_a;
+  const int *b = (const int *)_b;
+  return *a - *b;
+}
+
+#define MIN(_x, _y) ((_x) < (_y) ? (_x) : (_y))
+
+static int test_sort() {
+  int i, *e;
+  const int N = 10000;
+
+  srand(N);
+  for (i = 0; i < 1000; i++)
+    (void)rand();
+
+  struct allocator ptrlist_allocator;
+  dmrC_allocator_init(&ptrlist_allocator, "ptrlist_nodes", sizeof(struct ptr_list),
+	  __alignof__(struct ptr_list), CHUNK);
+  struct allocator int_allocator;
+  dmrC_allocator_init(&int_allocator, "ints", sizeof(int), __alignof__(int), CHUNK);
+  struct ptr_list *int_list = NULL;
+
+  for (i = 0; i < N; i++) {
+    e = (int*)dmrC_allocator_allocate(&int_allocator, 0);
+    *e = rand();
+    ptrlist_add(&int_list, e, &ptrlist_allocator);
+  }
+  if (ptrlist_size(int_list) != N)
+    return 1;
+  ptrlist_sort(&int_list, NULL, int_cmp);
+  // Sort already sorted stuff.
+  ptrlist_sort(&int_list, NULL, int_cmp);
+
+  int *p = NULL;
+  struct ptr_list_iter iter = ptrlist_forward_iterator(int_list);
+  int count = 0;
+  for (int *k = (int*)ptrlist_iter_next(&iter); k != NULL;
+       k = (int*)ptrlist_iter_next(&iter)) {
+    if (p != NULL) {
+      if (*k < *p)
+        return 1;
+    }
+    p = k;
+    count++;
+  }
+  if (count != N)
+    return 1;
+
+  struct ptr_list *l = int_list, *l2;
+  l2 = l;
+  int expected_count = 0;
+  do {
+    l2->nr_ = MIN(l2->nr_, rand() % 3);
+    for (i = 0; i < l2->nr_; i++) {
+      *((int *)(l2->list_[i])) = rand();
+      expected_count++;
+    }
+    l2 = l2->next_;
+  } while (l2 != l);
+  ptrlist_sort(&int_list, NULL, int_cmp);
+
+  p = NULL;
+  iter = ptrlist_forward_iterator(int_list);
+  count = 0;
+  for (int *k = (int*)ptrlist_iter_next(&iter); k != NULL;
+       k = (int*)ptrlist_iter_next(&iter)) {
+    if (p != NULL) {
+      if (*k < *p)
+        return 1;
+    }
+    p = k;
+    count++;
+  }
+  if (count != expected_count)
+    return 1;
+  ptrlist_remove_all(&int_list);
+  dmrC_allocator_destroy(&int_allocator);
+  dmrC_allocator_destroy(&ptrlist_allocator);
+  return 0;
+}
+
+struct mystruct {
+  int i;
+};
+
+struct mytoken {
+  const char *a;
+};
+
+static int test_ptrlist_basics() {
+	struct allocator ptrlist_allocator;
+	dmrC_allocator_init(&ptrlist_allocator, "ptrlist_nodes", sizeof(struct ptr_list),
+		__alignof__(struct ptr_list), CHUNK);
+	struct allocator token_allocator;
+	dmrC_allocator_init(&token_allocator, "ptr_list_tokens", sizeof(struct mytoken),
+		__alignof__(struct mytoken), CHUNK);
+	struct ptr_list *token_list = NULL;
+	if (ptrlist_size(token_list) != 0)
+		return 1;
+	struct mytoken *tok1 = (struct mytoken *)dmrC_allocator_allocate(&token_allocator, 0);
+	struct mytoken **tok1p = (struct mytoken **)ptrlist_add(&token_list, tok1, &ptrlist_allocator);
+	if (ptrlist_size(token_list) != 1)
+		return 1;
+	if (tok1 != *tok1p)
+		return 1;
+	if (ptrlist_first(token_list) != tok1)
+		return 1;
+	if (ptrlist_last(token_list) != tok1)
+		return 1;
+	struct mytoken *tok2 = (struct mytoken *)dmrC_allocator_allocate(&token_allocator, 0);
+	struct mytoken **tok2p = (struct mytoken **)ptrlist_add(&token_list, tok2, &ptrlist_allocator);
+	if (ptrlist_size(token_list) != 2)
+		return 1;
+	struct mytoken *tok3 = (struct mytoken *)dmrC_allocator_allocate(&token_allocator, 0);
+	ptrlist_add(&token_list, tok3, &ptrlist_allocator);
+	if (ptrlist_size(token_list) != 3)
+		return 1;
+	struct mytoken *tok4 = (struct mytoken *)dmrC_allocator_allocate(&token_allocator, 0);
+	ptrlist_add(&token_list, tok4, &ptrlist_allocator);
+	if (ptrlist_size(token_list) != 4)
+		return 1;
+	struct mytoken *tok5 = (struct mytoken *)dmrC_allocator_allocate(&token_allocator, 0);
+	struct mytoken **tok5p = (struct mytoken **)ptrlist_add(&token_list, tok5, &ptrlist_allocator);
+	if (ptrlist_size(token_list) != 5)
+		return 1;
+
+	if (tok2 != *tok2p)
+		return 1;
+	if (tok5 != *tok5p)
+		return 1;
+	if (ptrlist_first(token_list) != tok1)
+		return 1;
+	if (ptrlist_last(token_list) != tok5)
+		return 1;
+	struct mytoken *toks[5];
+	int lin1 = ptrlist_linearize(token_list, (void **)toks, 5);
+	if (lin1 != 5)
+		return 1;
+	if (toks[0] != tok1)
+		return 1;
+	if (toks[1] != tok2)
+		return 1;
+	if (toks[2] != tok3)
+		return 1;
+	if (toks[3] != tok4)
+		return 1;
+	if (toks[4] != tok5)
+		return 1;
+	if (ptrlist_size(token_list) != 5)
+		return 1;
+	ptrlist_pack(&token_list);
+	if (ptrlist_size(token_list) != 5)
+		return 1;
+
+	if (ptrlist_first(token_list) != tok1)
+		return 1;
+	if (ptrlist_last(token_list) != tok5)
+		return 1;
+
+	const int X = 5 + 1;
+	const int Y = X - 1;
+	const int Z = Y - 1;
+	struct ptr_list_iter iter1 = ptrlist_forward_iterator(token_list);
+	for (int i = 0; i < X; i++) {
+		struct mytoken *tk = (struct mytoken *)ptrlist_iter_next(&iter1);
+		if (tk == NULL) {
+			if (i == Y)
+				break;
+			return 1;
+		}
+		if (tk != toks[i])
+			return 1;
+	}
+	struct ptr_list_iter iter2 = ptrlist_reverse_iterator(token_list);
+	for (int i = 0; i < X; i++) {
+		struct mytoken *tk = (struct mytoken *)ptrlist_iter_prev(&iter2);
+		if (tk == NULL) {
+			if (i == Y)
+				break;
+			return 1;
+		}
+		if (tk != toks[Z - i])
+			return 1;
+	}
+	struct mytoken *tok0 = (struct mytoken *)dmrC_allocator_allocate(&token_allocator, 0);
+	struct ptr_list_iter iter3 = ptrlist_forward_iterator(token_list);
+	if (!ptrlist_iter_next(&iter3))
+		return 1;
+	ptrlist_iter_insert(&iter3, tok0);
+	if (ptrlist_size(token_list) != 6)
+		return 1;
+	if (ptrlist_first(token_list) != tok0)
+		return 1;
+	if (ptrlist_last(token_list) != tok5)
+		return 1;
+
+	struct allocator mystruct_allocator;
+	dmrC_allocator_init(&mystruct_allocator, "mystructs", sizeof(struct mystruct),
+		__alignof__(struct mystruct), CHUNK);
+	struct ptr_list *mystruct_list = NULL;
+
+	struct mystruct *s1 = (struct mystruct *)dmrC_allocator_allocate(&mystruct_allocator, 0);
+	s1->i = 1;
+	struct mystruct *s2 = (struct mystruct *)dmrC_allocator_allocate(&mystruct_allocator, 0);
+	s2->i = 2;
+	struct mystruct *s3 = (struct mystruct *)dmrC_allocator_allocate(&mystruct_allocator, 0);
+	s3->i = 3;
+	struct mystruct *s4 = (struct mystruct *)dmrC_allocator_allocate(&mystruct_allocator, 0);
+	s4->i = 4;
+	struct mystruct *s5 = (struct mystruct *)dmrC_allocator_allocate(&mystruct_allocator, 0);
+	s5->i = 5;
+	struct mystruct *s6 = (struct mystruct *)dmrC_allocator_allocate(&mystruct_allocator, 0);
+	s6->i = 6;
+
+	ptrlist_add(&mystruct_list, s1, &ptrlist_allocator);
+	ptrlist_add(&mystruct_list, s2, &ptrlist_allocator);
+	ptrlist_add(&mystruct_list, s3, &ptrlist_allocator);
+	ptrlist_add(&mystruct_list, s4, &ptrlist_allocator);
+	ptrlist_add(&mystruct_list, s5, &ptrlist_allocator);
+	ptrlist_add(&mystruct_list, s6, &ptrlist_allocator);
+
+	struct mystruct *serial1_expected[6] = { s1, s2, s3, s4, s5, s6 };
+	struct mystruct *serial1_got[6];
+	ptrlist_linearize(mystruct_list, (void **)serial1_got, 6);
+	for (int i = 0; i < 6; i++) {
+		if (serial1_expected[i] != serial1_got[i])
+			return 1;
+	}
+
+	if (ptrlist_remove(&mystruct_list, s3, 1) != 0)
+		return 1;
+	struct ptr_list_iter iter4 = ptrlist_forward_iterator(mystruct_list);
+	for (struct mystruct *p = (struct mystruct *)ptrlist_iter_next(&iter4); p != NULL;
+		p = (struct mystruct *)ptrlist_iter_next(&iter4)) {
+		if (p->i == 4)
+			ptrlist_iter_remove(&iter4);
+	}
+	if (ptrlist_size(mystruct_list) != 4)
+		return 1;
+
+	struct mystruct *serial3_expected[4] = { s1, s2, s5, s6 };
+	struct mystruct *serial3_got[4];
+	int reverse_expected[2] = { 2, 1 };
+
+	int i = 0;
+	struct mystruct *p;
+	FOR_EACH_PTR(mystruct_list, p) {
+		if (i == 4)
+			return 1;
+		serial3_got[i++] = p;
+		if (i == 3) {
+			struct mystruct *p2;
+			int j = 0;
+			RECURSE_PTR_REVERSE(p, p2) {
+				if (j >= 2 || reverse_expected[j] != p2->i)
+					return 1;
+				j++;
+			} END_FOR_EACH_PTR_REVERSE(p2);
+		}
+	} END_FOR_EACH_PTR(p);
+	if (i != 4)
+		return 1;
+	for (int i = 0; i < 4; i++) {
+		if (serial3_expected[i] != serial3_got[i])
+			return 1;
+	}
+
+	i = 0;
+	PREPARE_PTR_LIST(mystruct_list, p);
+	while (p != NULL) {
+		if (i == 4)
+			return 1;
+		serial3_got[i++] = p;
+		NEXT_PTR_LIST(p);
+	}
+	FINISH_PTR_LIST(p);
+	if (i != 4)
+		return 1;
+	for (int i = 0; i < 4; i++) {
+		if (serial3_expected[i] != serial3_got[i])
+			return 1;
+	}
+
+	i = 0;
+	FOR_EACH_PTR_REVERSE(mystruct_list, p) {
+		if (i == 4)
+			return 1;
+		serial3_got[i++] = p;
+		if (i == 2) {
+			struct mystruct *p3;
+			int j = 0;
+			RECURSE_PTR_REVERSE(p, p3) {
+				if (j >= 2 || reverse_expected[j] != p3->i)
+					return 1;
+				j++;
+			} END_FOR_EACH_PTR_REVERSE(p3);
+		}
+	} END_FOR_EACH_PTR_REVERSE(p);
+	if (i != 4)
+		return 1;
+	for (int i = 0; i < 4; i++) {
+		if (serial3_expected[3-i] != serial3_got[i])
+			return 1;
+	}
+
+	ptrlist_remove_all(&token_list);
+	ptrlist_remove_all(&mystruct_list);
+
+	dmrC_allocator_destroy(&token_allocator);
+	dmrC_allocator_destroy(&mystruct_allocator);
+	dmrC_allocator_destroy(&ptrlist_allocator);
+	return 0;
+}
+
+int test_ptrlist() {
+	if (test_sort() != 0)
+		return 1;
+
+	/* For testing we set N_ temporarily */
+	N_ = 2;
+	int failure_count = test_ptrlist_basics();
+	N_ = LIST_NODE_NR;
+
+	if (failure_count == 0)
+	  printf("ptrlist test okay\n");
+  return failure_count;
+}

dmr_c/src/ptrlist.h

@@ -0,0 +1,333 @@
+#ifndef DMR_C_PTRLIST_H
+#define DMR_C_PTRLIST_H
+
+/*
+* Generic pointer list manipulation code.
+*
+* (C) Copyright Linus Torvalds 2003-2005
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <allocate.h>
+
+#include <assert.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*
+* The ptr list data structure is like a train - with cars linked to each other.
+* Just as in a train each car has many seats, so in ptr list each "node" has
+* several entries. Unlike a train however, the ptr list is arranged as a ring,
+* i.e. the the front and back nodes are linked to each other. Hence there is no
+* such thing as a 'head' of the list - i.e. any node can be the head!
+*/
+
+#ifndef LIST_NODE_NR
+#define LIST_NODE_NR (29)
+#endif
+
+#define DECLARE_PTR_LIST(listname, type)                                       \
+	struct listname {                                                      \
+		int nr_ : 8;                                                   \
+		int rm_ : 8;                                                   \
+		struct listname *prev_;                                        \
+		struct listname *next_;                                        \
+		struct allocator *allocator_;                                  \
+		type *list_[LIST_NODE_NR];                                     \
+	}
+
+/* Each node in the list */
+DECLARE_PTR_LIST(ptr_list, void);
+
+struct ptr_list_iter {
+	struct ptr_list *__head;
+	struct ptr_list *__list;
+	int __nr;
+};
+
+/* The ptr list */
+extern int ptrlist_size(const struct ptr_list *self);
+extern void **ptrlist_add(struct ptr_list **self, void *ptr, struct allocator *ptr_list_allocator);
+extern void *ptrlist_nth_entry(struct ptr_list *list, unsigned int idx);
+extern void *ptrlist_first(struct ptr_list *list);
+extern void *ptrlist_last(struct ptr_list *list);
+extern int ptrlist_linearize(struct ptr_list *head, void **arr, int max);
+extern void ptrlist_split_node(struct ptr_list *head);
+extern void ptrlist_pack(struct ptr_list **self);
+extern void ptrlist_remove_all(struct ptr_list **self);
+extern int ptrlist_remove(struct ptr_list **self, void *entry, int count);
+extern int ptrlist_replace(struct ptr_list **self, void *old_ptr, void *new_ptr,
+	int count);
+extern void *ptrlist_undo_last(struct ptr_list **self);
+extern void *ptrlist_delete_last(struct ptr_list **self);
+extern void ptrlist_concat(struct ptr_list *a, struct ptr_list **self);
+extern void ptrlist_sort(struct ptr_list **self, void *,
+			 int (*cmp)(void *, const void *, const void *));
+
+/* iterator functions */
+extern struct ptr_list_iter ptrlist_forward_iterator(struct ptr_list *self);
+extern struct ptr_list_iter ptrlist_reverse_iterator(struct ptr_list *self);
+extern void *ptrlist_iter_next(struct ptr_list_iter *self);
+extern void *ptrlist_iter_prev(struct ptr_list_iter *self);
+extern void ptrlist_iter_split_current(struct ptr_list_iter *self);
+extern void ptrlist_iter_insert(struct ptr_list_iter *self, void *newitem);
+extern void ptrlist_iter_remove(struct ptr_list_iter *self);
+extern void ptrlist_iter_set(struct ptr_list_iter *self, void *ptr);
+extern void ptrlist_iter_mark_deleted(struct ptr_list_iter *self);
+
+static inline void **ptrlist_iter_this_address(struct ptr_list_iter *self) {
+	return &self->__list->list_[self->__nr];
+}
+#define ptr_list_empty(x) ((x) == NULL)
+#define PTR_ENTRY_NOTAG(h,i)	((h)->list_[i])
+#define PTR_ENTRY(h,i)	(void *)(PTR_ENTRY_NOTAG(h,i))
+
+#if 1
+
+#define FOR_EACH_PTR(list, var) \
+	{ struct ptr_list_iter var##iter__ = ptrlist_forward_iterator((struct ptr_list *)list); \
+	for (var = ptrlist_iter_next(&var##iter__); var != NULL; var = ptrlist_iter_next(&var##iter__))
+#define END_FOR_EACH_PTR(var) }
+
+#define FOR_EACH_PTR_REVERSE(list, var) \
+	{ struct ptr_list_iter var##iter__ = ptrlist_reverse_iterator((struct ptr_list *)list); \
+	for (var = ptrlist_iter_prev(&var##iter__); var != NULL; var = ptrlist_iter_prev(&var##iter__))
+#define END_FOR_EACH_PTR_REVERSE(var) }
+
+#define RECURSE_PTR_REVERSE(list, var) \
+	{ struct ptr_list_iter var##iter__ = list##iter__; \
+	for (var = ptrlist_iter_prev(&var##iter__); var != NULL; var = ptrlist_iter_prev(&var##iter__))
+
+#define PREPARE_PTR_LIST(list, var)	\
+	struct ptr_list_iter var##iter__ = ptrlist_forward_iterator((struct ptr_list *)list); \
+	var = ptrlist_iter_next(&var##iter__)
+
+#define NEXT_PTR_LIST(var) \
+	var = ptrlist_iter_next(&var##iter__)
+#define FINISH_PTR_LIST(var) 
+
+#define THIS_ADDRESS(type, var) \
+	(type *)ptrlist_iter_this_address(&var##iter__)
+
+#define DELETE_CURRENT_PTR(var) \
+	ptrlist_iter_remove(&var##iter__)	
+
+#define REPLACE_CURRENT_PTR(type, var, replacement) \
+	ptrlist_iter_set(&var##iter__, replacement)
+
+#define INSERT_CURRENT(newval, var) \
+	ptrlist_iter_insert(&var##iter__, newval)	
+
+#define MARK_CURRENT_DELETED(PTR_TYPE, var) \
+	ptrlist_iter_mark_deleted(&var##iter__)
+
+#else
+
+#define DO_PREPARE(head, ptr, __head, __list, __nr, PTR_ENTRY)				\
+	do {										\
+		struct ptr_list *__head = (struct ptr_list *) (head);			\
+		struct ptr_list *__list = __head;					\
+		int __nr = 0;								\
+		if (__head) ptr = PTR_ENTRY(__head, 0);					\
+		else ptr = NULL
+
+#define DO_NEXT(ptr, __head, __list, __nr, PTR_ENTRY)					\
+		if (ptr) {								\
+			if (++__nr < __list->nr_) {					\
+				ptr = PTR_ENTRY(__list,__nr);				\
+			} else {							\
+				__list = __list->next_;					\
+				ptr = NULL;						\
+				while (__list->nr_ == 0 && __list != __head)		\
+					__list = __list->next_;				\
+				if (__list != __head) {					\
+					__nr = 0;					\
+					ptr = PTR_ENTRY(__list,0);			\
+				}							\
+			}								\
+		}
+
+#define DO_RESET(ptr, __head, __list, __nr, PTR_ENTRY)					\
+	do {										\
+		__nr = 0;								\
+		__list = __head;							\
+		if (__head) ptr = PTR_ENTRY(__head, 0);					\
+	} while (0)
+
+#define DO_FINISH(ptr, __head, __list, __nr)						\
+		(void)(__nr); /* Sanity-check nesting */				\
+	} while (0)
+
+#define PREPARE_PTR_LIST(head, ptr) \
+	DO_PREPARE(head, ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY)
+
+#define NEXT_PTR_LIST(ptr) \
+	DO_NEXT(ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY)
+
+#define RESET_PTR_LIST(ptr) \
+	DO_RESET(ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY)
+
+#define FINISH_PTR_LIST(ptr) \
+	DO_FINISH(ptr, __head##ptr, __list##ptr, __nr##ptr)
+
+#define DO_FOR_EACH(head, ptr, __head, __list, __nr, PTR_ENTRY) do {			\
+	struct ptr_list *__head = (struct ptr_list *) (head);				\
+	struct ptr_list *__list = __head;						\
+	if (__head) {									\
+		do { int __nr;								\
+			for (__nr = 0; __nr < __list->nr_; __nr++) {			\
+				do {							\
+					ptr = PTR_ENTRY(__list,__nr);			\
+					if (__list->rm_ && !ptr)				\
+						continue;				\
+					do {
+
+#define DO_END_FOR_EACH(ptr, __head, __list, __nr)					\
+					} while (0);					\
+				} while (0);						\
+			}								\
+		} while ((__list = __list->next_) != __head);				\
+	}										\
+} while (0)
+
+#define DO_FOR_EACH_REVERSE(head, ptr, __head, __list, __nr, PTR_ENTRY) do {		\
+	struct ptr_list *__head = (struct ptr_list *) (head);				\
+	struct ptr_list *__list = __head;						\
+	if (__head) {									\
+		do { int __nr;								\
+			__list = __list->prev_;						\
+			__nr = __list->nr_;						\
+			while (--__nr >= 0) {						\
+				do {							\
+					ptr = PTR_ENTRY(__list,__nr);			\
+					if (__list->rm_ && !ptr)				\
+						continue;				\
+					do {
+
+
+#define DO_END_FOR_EACH_REVERSE(ptr, __head, __list, __nr)				\
+					} while (0);					\
+				} while (0);						\
+			}								\
+		} while (__list != __head);						\
+	}										\
+} while (0)
+
+#define DO_REVERSE(ptr, __head, __list, __nr, new, __newhead,				\
+		   __newlist, __newnr, PTR_ENTRY) do { 					\
+	struct ptr_list *__newhead = __head;						\
+	struct ptr_list *__newlist = __list;						\
+	int __newnr = __nr;								\
+	new = ptr;									\
+	goto __inside##new;								\
+	if (1) {									\
+		do {									\
+			__newlist = __newlist->prev_;					\
+			__newnr = __newlist->nr_;					\
+	__inside##new:									\
+			while (--__newnr >= 0) {					\
+				do {							\
+					new = PTR_ENTRY(__newlist,__newnr);		\
+					do {
+
+#define RECURSE_PTR_REVERSE(ptr, new)							\
+	DO_REVERSE(ptr, __head##ptr, __list##ptr, __nr##ptr,				\
+		   new, __head##new, __list##new, __nr##new, PTR_ENTRY)
+
+#define DO_THIS_ADDRESS(PTR_TYPE, ptr, __head, __list, __nr)					\
+	((PTR_TYPE*) (__list->list_ + __nr))
+
+
+#define FOR_EACH_PTR(head, ptr) \
+	DO_FOR_EACH(head, ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY)
+
+#define END_FOR_EACH_PTR(ptr) \
+	DO_END_FOR_EACH(ptr, __head##ptr, __list##ptr, __nr##ptr)
+
+#define FOR_EACH_PTR_NOTAG(head, ptr) \
+	DO_FOR_EACH(head, ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY_NOTAG)
+
+#define END_FOR_EACH_PTR_NOTAG(ptr) END_FOR_EACH_PTR(ptr)
+
+#define FOR_EACH_PTR_REVERSE(head, ptr) \
+	DO_FOR_EACH_REVERSE(head, ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY)
+
+#define END_FOR_EACH_PTR_REVERSE(ptr) \
+	DO_END_FOR_EACH_REVERSE(ptr, __head##ptr, __list##ptr, __nr##ptr)
+
+#define FOR_EACH_PTR_REVERSE_NOTAG(head, ptr) \
+	DO_FOR_EACH_REVERSE(head, ptr, __head##ptr, __list##ptr, __nr##ptr, PTR_ENTRY_NOTAG)
+
+#define END_FOR_EACH_PTR_REVERSE_NOTAG(ptr) END_FOR_EACH_PTR_REVERSE(ptr)
+
+#define THIS_ADDRESS(PTR_TYPE, ptr) \
+	DO_THIS_ADDRESS(PTR_TYPE, ptr, __head##ptr, __list##ptr, __nr##ptr)
+
+
+#define DO_SPLIT(ptr, __head, __list, __nr) do {					\
+	ptrlist_split_node(__list);							\
+	if (__nr >= __list->nr_) {							\
+		__nr -= __list->nr_;							\
+		__list = __list->next_;							\
+	};										\
+} while (0)
+
+#define DO_INSERT_CURRENT(new, ptr, __head, __list, __nr) do {				\
+	void **__this, **__last;							\
+	if (__list->nr_ == LIST_NODE_NR)							\
+		DO_SPLIT(ptr, __head, __list, __nr);					\
+	__this = __list->list_ + __nr;							\
+	__last = __list->list_ + __list->nr_ - 1;						\
+	while (__last >= __this) {							\
+		__last[1] = __last[0];							\
+		__last--;								\
+	}										\
+	*__this = (new);								\
+	__list->nr_++;									\
+} while (0)
+
+#define INSERT_CURRENT(new, ptr) \
+	DO_INSERT_CURRENT(new, ptr, __head##ptr, __list##ptr, __nr##ptr)
+
+#define DO_DELETE_CURRENT(ptr, __head, __list, __nr) do {				\
+	void **__this = __list->list_ + __nr;						\
+	void **__last = __list->list_ + __list->nr_ - 1;					\
+	while (__this < __last) {							\
+		__this[0] = __this[1];							\
+		__this++;								\
+	}										\
+	*__this = (void *)((uintptr_t)0xf0f0f0f0);							\
+	__list->nr_--; __nr--;								\
+} while (0)
+
+#define DELETE_CURRENT_PTR(ptr) \
+	DO_DELETE_CURRENT(ptr, __head##ptr, __list##ptr, __nr##ptr)
+
+#define REPLACE_CURRENT_PTR(PTR_TYPE, ptr, new_ptr)						\
+	do { *THIS_ADDRESS(PTR_TYPE, ptr) = (new_ptr); } while (0)
+
+#define DO_MARK_CURRENT_DELETED(PTR_TYPE, ptr, __list) do {	\
+		REPLACE_CURRENT_PTR(PTR_TYPE, ptr, NULL);		\
+		__list->rm++;				\
+	} while (0)
+
+#define MARK_CURRENT_DELETED(PTR_TYPE, ptr) \
+	DO_MARK_CURRENT_DELETED(PTR_TYPE, ptr, __list##ptr)
+	
+#endif
+
+extern int test_ptrlist();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/runtests.c

@@ -0,0 +1,20 @@
+#include <allocate.h>
+#include <ptrlist.h>
+#include <token.h>
+#include <parse.h>
+#include <stdio.h>
+
+int main()
+{
+	int failure_count = 0;
+	failure_count += dmrC_test_allocator();
+	failure_count += test_ptrlist();
+	//failure_count += test_tokenizer();
+	//failure_count += dmrC_test_parse();
+
+	if (failure_count == 0)
+		printf("Tests OK\n");
+	else
+		printf("Tests FAILED\n");
+	return failure_count == 0 ? 0 : 1;
+}

dmr_c/src/scope.c

@@ -0,0 +1,162 @@
+/*
+ * Symbol scoping.
+ *
+ * This is pretty trivial.
+ *
+ * Copyright (C) 2003 Transmeta Corp.
+ *               2003-2004 Linus Torvalds
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+ /*
+ * This version is part of the dmr_c project.
+ * Copyright (C) 2017 Dibyendu Majumdar
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <port.h>
+#include <lib.h>
+#include <allocate.h>
+#include <symbol.h>
+#include <scope.h>
+
+void dmrC_init_scope(struct dmr_C *C) {
+	struct scope *scope = (struct scope *)dmrC_allocator_allocate(&C->scope_allocator, 0);
+	memset(scope, 0, sizeof(*scope));
+	scope->next = scope;
+	C->builtin_scope = scope;
+	C->block_scope = C->builtin_scope;		// regular automatic variables etc
+	C->function_scope = C->builtin_scope;	// labels, arguments etc
+	C->file_scope = C->builtin_scope;		// static
+	C->global_scope = C->builtin_scope;		// externally visible
+}
+
+void dmrC_destroy_all_scopes(struct dmr_C *C) {
+        (void) C;
+}
+
+void dmrC_bind_scope(struct dmr_C *C, struct symbol *sym, struct scope *scope)
+{
+	sym->scope = scope;
+	dmrC_add_symbol(C, &scope->symbols, sym);
+}
+
+void dmrC_rebind_scope(struct dmr_C *C, struct symbol *sym, struct scope *news)
+{
+	struct scope *old = sym->scope;
+
+	if (old == news)
+		return;
+
+	if (old)
+		ptrlist_remove((struct ptr_list **) &old->symbols, sym, 1);
+
+	dmrC_bind_scope(C, sym, news);
+}
+
+static void start_scope(struct dmr_C *C, struct scope **s)
+{
+	struct scope *scope = (struct scope *)dmrC_allocator_allocate(&C->scope_allocator, 0);
+	memset(scope, 0, sizeof(*scope));
+	scope->next = *s;
+	*s = scope;
+}
+
+void dmrC_start_file_scope(struct dmr_C *C)
+{
+	struct scope *scope = (struct scope *)dmrC_allocator_allocate(&C->scope_allocator, 0);
+
+	memset(scope, 0, sizeof(*scope));
+	scope->next = C->builtin_scope;
+	C->file_scope = scope;
+
+	/* top-level stuff defaults to file scope, "extern" etc will choose global scope */
+	C->function_scope = scope;
+	C->block_scope = scope;
+}
+
+void dmrC_start_symbol_scope(struct dmr_C *C)
+{
+	start_scope(C, &C->block_scope);
+}
+
+void dmrC_start_function_scope(struct dmr_C *C)
+{
+	start_scope(C, &C->function_scope);
+	start_scope(C, &C->block_scope);
+}
+
+static void remove_symbol_scope(struct dmr_C *C, struct symbol *sym)
+{
+        (void) C;
+	struct symbol **ptr = &sym->ident->symbols;
+
+	while (*ptr != sym)
+		ptr = &(*ptr)->next_id;
+	*ptr = sym->next_id;
+}
+
+static void end_scope(struct dmr_C *C, struct scope **s)
+{
+	struct scope *scope = *s;
+	struct symbol_list *symbols = scope->symbols;
+	struct symbol *sym;
+
+	*s = scope->next;
+	scope->symbols = NULL;
+	FOR_EACH_PTR(symbols, sym) {
+		remove_symbol_scope(C, sym);
+	} END_FOR_EACH_PTR(sym);
+}
+
+void dmrC_end_file_scope(struct dmr_C *C)
+{
+	end_scope(C, &C->file_scope);
+}
+
+void dmrC_new_file_scope(struct dmr_C *C)
+{
+	if (C->file_scope != C->builtin_scope)
+		dmrC_end_file_scope(C);
+	dmrC_start_file_scope(C);
+}
+
+void dmrC_end_symbol_scope(struct dmr_C *C)
+{
+	end_scope(C, &C->block_scope);
+}
+
+void dmrC_end_function_scope(struct dmr_C *C)
+{
+	end_scope(C, &C->block_scope);
+	end_scope(C, &C->function_scope);
+}
+
+int dmrC_is_outer_scope(struct dmr_C *C, struct scope *scope)
+{
+	if (scope == C->block_scope)
+		return 0;
+	if (scope == C->builtin_scope && C->block_scope->next == C->builtin_scope)
+		return 0;
+	return 1;
+}
+

dmr_c/src/scope.h

@@ -0,0 +1,73 @@
+#ifndef DMR_C_SCOPE_H
+#define DMR_C_SCOPE_H
+/*
+* Symbol scoping is pretty simple.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <lib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct scope {
+	struct token *token;    /* Scope start information */
+	struct symbol_list *symbols; /* List of symbols in this scope */
+	struct scope *next;
+};
+
+static inline int dmrC_toplevel(struct dmr_C *C, struct scope *scope)
+{
+	return scope == C->file_scope || scope == C->global_scope;
+}
+
+extern void dmrC_start_file_scope(struct dmr_C *C);
+extern void dmrC_end_file_scope(struct dmr_C *C);
+extern void dmrC_new_file_scope(struct dmr_C *C);
+
+extern void dmrC_start_symbol_scope(struct dmr_C *C);
+extern void dmrC_end_symbol_scope(struct dmr_C *C);
+
+extern void dmrC_start_function_scope(struct dmr_C *C);
+extern void dmrC_end_function_scope(struct dmr_C *C);
+
+extern void dmrC_bind_scope(struct dmr_C *C, struct symbol *, struct scope *);
+extern void dmrC_rebind_scope(struct dmr_C *C, struct symbol *sym, struct scope *news);
+
+extern int dmrC_is_outer_scope(struct dmr_C *C, struct scope *);
+
+extern void dmrC_init_scope(struct dmr_C *C);
+extern void dmrC_destroy_all_scopes(struct dmr_C *C);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/symbol.c

@@ -0,0 +1,827 @@
+/*
+ * Symbol lookup and handling.
+ *
+ * Copyright (C) 2003 Transmeta Corp.
+ *               2003-2004 Linus Torvalds
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+ /*
+ * This version is part of the dmr_c project.
+ * Copyright (C) 2017 Dibyendu Majumdar
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <port.h>
+#include <lib.h>
+#include <allocate.h>
+#include <ptrlist.h>
+#include <token.h>
+#include <parse.h>
+#include <symbol.h>
+#include <scope.h>
+#include <expression.h>
+
+#include <target.h>
+
+/*
+ * If the symbol is an inline symbol, add it to the list of symbols to parse
+ */
+void dmrC_access_symbol(struct global_symbols_t *S, struct symbol *sym)
+{
+	if (sym->ctype.modifiers & MOD_INLINE) {
+		if (!(sym->ctype.modifiers & MOD_ACCESSED)) {
+			dmrC_add_symbol(S->C, &S->translation_unit_used_list, sym);
+			sym->ctype.modifiers |= MOD_ACCESSED;
+		}
+	}
+}
+
+struct symbol *dmrC_lookup_symbol(struct ident *ident, enum namespace_type ns)
+{
+	struct symbol *sym;
+
+	for (sym = ident->symbols; sym; sym = sym->next_id) {
+		if (sym->ns & ns) {
+			sym->used = 1;
+			return sym;
+		}
+	}
+	return NULL;
+}
+
+struct context *dmrC_alloc_context(struct global_symbols_t *S)
+{
+	return (struct context *)dmrC_allocator_allocate(&S->context_allocator, 0);
+}
+
+struct symbol *dmrC_alloc_symbol(struct global_symbols_t *S, struct position pos, int type)
+{
+	struct symbol *sym = (struct symbol *) dmrC_allocator_allocate(&S->symbol_allocator, 0);
+	sym->type = type;
+	sym->pos = pos;
+	sym->endpos.type = 0;
+	return sym;
+}
+
+struct struct_union_info {
+	unsigned long max_align;
+	unsigned long bit_size;
+	int align_size;
+};
+
+/*
+ * Unions are fairly easy to lay out ;)
+ */
+static void lay_out_union(struct global_symbols_t *S, struct symbol *sym, struct struct_union_info *info)
+{
+	dmrC_examine_symbol_type(S, sym);
+
+	// Unnamed bitfields do not affect alignment.
+	if (sym->ident || !dmrC_is_bitfield_type(sym)) {
+		if (sym->ctype.alignment > info->max_align)
+			info->max_align = sym->ctype.alignment;
+	}
+
+	if (sym->bit_size > (int) info->bit_size)
+		info->bit_size = sym->bit_size;
+
+	sym->offset = 0;
+}
+
+static int bitfield_base_size(struct symbol *sym)
+{
+	if (sym->type == SYM_NODE)
+		sym = sym->ctype.base_type;
+	if (sym->type == SYM_BITFIELD)
+		sym = sym->ctype.base_type;
+	return sym->bit_size;
+}
+
+/*
+ * Structures are a bit more interesting to lay out
+ */
+static void lay_out_struct(struct global_symbols_t *S, struct symbol *sym, struct struct_union_info *info)
+{
+	unsigned long bit_size, align_bit_mask;
+	int base_size;
+
+	dmrC_examine_symbol_type(S, sym);
+
+	// Unnamed bitfields do not affect alignment.
+	if (sym->ident || !dmrC_is_bitfield_type(sym)) {
+		if (sym->ctype.alignment > info->max_align)
+			info->max_align = sym->ctype.alignment;
+	}
+
+	bit_size = info->bit_size;
+	base_size = sym->bit_size; 
+
+	/*
+	 * Unsized arrays cause us to not align the resulting
+	 * structure size
+	 */
+	if (base_size < 0) {
+		info->align_size = 0;
+		base_size = 0;
+	}
+
+	align_bit_mask = dmrC_bytes_to_bits(S->C->target, sym->ctype.alignment) - 1;
+
+	/*
+	 * Bitfields have some very special rules..
+	 */
+	if (dmrC_is_bitfield_type (sym)) {
+		unsigned long bit_offset = bit_size & align_bit_mask;
+		int room = bitfield_base_size(sym) - bit_offset;
+		// Zero-width fields just fill up the unit.
+		int width = base_size ? base_size : (bit_offset ? room : 0);
+
+		if (width > room) {
+			bit_size = (bit_size + align_bit_mask) & ~align_bit_mask;
+			bit_offset = 0;
+		}
+		sym->offset = dmrC_bits_to_bytes(S->C->target, bit_size - bit_offset);
+		sym->bit_offset = bit_offset;
+		sym->ctype.base_type->bit_offset = bit_offset;
+		info->bit_size = bit_size + width;
+		// dmrC_warning (sym->pos, "bitfield: offset=%d:%d  size=:%d", sym->offset, sym->bit_offset, width);
+
+		return;
+	}
+
+	/*
+	 * Otherwise, just align it right and add it up..
+	 */
+	bit_size = (bit_size + align_bit_mask) & ~align_bit_mask;
+	sym->offset = dmrC_bits_to_bytes(S->C->target, bit_size);
+
+	info->bit_size = bit_size + base_size;
+	// dmrC_warning (sym->pos, "regular: offset=%d", sym->offset);
+}
+
+static struct symbol * examine_struct_union_type(struct global_symbols_t *S, struct symbol *sym, int advance)
+{
+	struct struct_union_info info = {
+		.max_align = 1,
+		.bit_size = 0,
+		.align_size = 1
+	};
+	unsigned long bit_size, bit_align;
+	void (*fn)(struct global_symbols_t *S, struct symbol *, struct struct_union_info *);
+	struct symbol *member;
+
+	fn = advance ? lay_out_struct : lay_out_union;
+	FOR_EACH_PTR(sym->symbol_list, member) {
+		fn(S, member, &info);
+	} END_FOR_EACH_PTR(member); 
+
+	if (!sym->ctype.alignment)
+		sym->ctype.alignment = info.max_align;
+	bit_size = info.bit_size;
+	if (info.align_size) {
+		bit_align = dmrC_bytes_to_bits(S->C->target, sym->ctype.alignment)-1;
+		bit_size = (bit_size + bit_align) & ~bit_align;
+	}
+	sym->bit_size = bit_size;
+	return sym;
+}
+
+static struct symbol *examine_base_type(struct global_symbols_t *S, struct symbol *sym)
+{
+	struct symbol *base_type;
+
+	/* Check the base type */
+	base_type = dmrC_examine_symbol_type(S, sym->ctype.base_type);
+	if (!base_type || base_type->type == SYM_PTR)
+		return base_type;
+	sym->ctype.as |= base_type->ctype.as;
+	sym->ctype.modifiers |= base_type->ctype.modifiers & MOD_PTRINHERIT;
+	dmrC_concat_context_list(base_type->ctype.contexts,
+				 &sym->ctype.contexts);
+	if (base_type->type == SYM_NODE) {
+		base_type = base_type->ctype.base_type;
+		sym->ctype.base_type = base_type;
+	}
+	return base_type;
+}
+
+static struct symbol * examine_array_type(struct global_symbols_t *S, struct symbol *sym)
+{
+	struct symbol *base_type = examine_base_type(S, sym);
+	unsigned int bit_size = -1, alignment;
+	struct expression *array_size = sym->array_size;
+
+	if (!base_type)
+		return sym;
+
+	if (array_size) {	
+		bit_size = (unsigned int) dmrC_array_element_offset(S->C->target, base_type->bit_size,
+						(int) dmrC_get_expression_value_silent(S->C, array_size));
+		if (array_size->type != EXPR_VALUE) {
+			if (S->C->Wvla)
+				dmrC_warning(S->C, array_size->pos, "Variable length array is used.");
+			bit_size = -1;
+		}
+	}
+	alignment = base_type->ctype.alignment;
+	if (!sym->ctype.alignment)
+		sym->ctype.alignment = alignment;
+	sym->bit_size = bit_size;
+	return sym;
+}
+
+static struct symbol *examine_bitfield_type(struct global_symbols_t *S, struct symbol *sym)
+{
+	struct symbol *base_type = examine_base_type(S, sym);
+	unsigned long bit_size, alignment, modifiers;
+
+	if (!base_type)
+		return sym;
+	bit_size = base_type->bit_size;
+	if (sym->bit_size > (int) bit_size)
+		dmrC_warning(S->C, sym->pos, "impossible field-width, %d, for this type",  sym->bit_size);
+
+	alignment = base_type->ctype.alignment;
+	if (!sym->ctype.alignment)
+		sym->ctype.alignment = alignment;
+	modifiers = base_type->ctype.modifiers;
+
+	/* Bitfields are unsigned, unless the base type was explicitly signed */
+	if (!(modifiers & MOD_EXPLICITLY_SIGNED))
+		modifiers = (modifiers & ~MOD_SIGNED) | MOD_UNSIGNED;
+	sym->ctype.modifiers |= modifiers & MOD_SIGNEDNESS;
+	return sym;
+}
+
+/*
+ * "typeof" will have to merge the types together
+ */
+void dmrC_merge_type(struct symbol *sym, struct symbol *base_type)
+{
+	sym->ctype.as |= base_type->ctype.as;
+	sym->ctype.modifiers |= (base_type->ctype.modifiers & ~MOD_STORAGE);
+	dmrC_concat_context_list(base_type->ctype.contexts,
+				 &sym->ctype.contexts);
+	sym->ctype.base_type = base_type->ctype.base_type;
+	if (sym->ctype.base_type->type == SYM_NODE)
+		dmrC_merge_type(sym, sym->ctype.base_type);
+}
+
+static int count_array_initializer(struct global_symbols_t *S, struct symbol *t, struct expression *expr)
+{
+	int nr = 0;
+	int is_char = 0;
+
+	/*
+	 * Arrays of character types are special; they can be initialized by
+	 * string literal _or_ by string literal in braces.  The latter means
+	 * that with T x[] = {<string literal>} number of elements in x depends
+	 * on T - if it's a character type, we get the length of string literal
+	 * (including NUL), otherwise we have one element here.
+	 */
+	if (t->ctype.base_type == &S->int_type && t->ctype.modifiers & MOD_CHAR)
+		is_char = 1;
+
+	switch (expr->type) {
+	case EXPR_INITIALIZER: {
+		struct expression *entry;
+		int count = 0;
+		int str_len = 0;
+		FOR_EACH_PTR(expr->expr_list, entry) {
+			count++;
+			switch (entry->type) {
+			case EXPR_INDEX:
+				if ((int)entry->idx_to >= nr)
+					nr = entry->idx_to+1;
+				break;
+			case EXPR_PREOP: {
+				struct expression *e = entry;
+				if (is_char) {
+					while (e && e->type == EXPR_PREOP && e->op == '(')
+						e = e->unop;
+					if (e && e->type == EXPR_STRING) {
+						entry = e;
+			case EXPR_STRING:
+						if (is_char)
+							str_len = entry->string->length;
+					}
+
+
+				}
+			}
+			default:
+				nr++;
+			}
+		} END_FOR_EACH_PTR(entry);
+		if (count == 1 && str_len)
+			nr = str_len;
+		break;
+	}
+	case EXPR_PREOP:
+		if (is_char) { 
+			struct expression *e = expr;
+			while (e && e->type == EXPR_PREOP && e->op == '(')
+				e = e->unop;
+			if (e && e->type == EXPR_STRING) {
+				expr = e;
+	case EXPR_STRING:
+				if (is_char)
+					nr = expr->string->length;
+			}
+		}
+		break;
+	default:
+		break;
+	}
+	return nr;
+}
+
+static struct expression *get_symbol_initializer(struct symbol *sym)
+{
+	do {
+		if (sym->initializer)
+			return sym->initializer;
+	} while ((sym = sym->same_symbol) != NULL);
+	return NULL;
+}
+static struct symbol * examine_node_type(struct global_symbols_t *S, struct symbol *sym)
+{
+	struct symbol *base_type = examine_base_type(S, sym);
+	int bit_size;
+	unsigned long alignment;
+
+	/* SYM_NODE - figure out what the type of the node was.. */
+	bit_size = 0;
+	alignment = 0;
+	if (!base_type)
+		return sym;
+
+	bit_size = base_type->bit_size;
+	alignment = base_type->ctype.alignment;
+
+	/* Pick up signedness information into the node */
+	sym->ctype.modifiers |= (MOD_SIGNEDNESS & base_type->ctype.modifiers);
+
+	if (!sym->ctype.alignment)
+		sym->ctype.alignment = alignment;
+
+	/* Unsized array? The size might come from the initializer.. */
+	if (bit_size < 0 && base_type->type == SYM_ARRAY) {
+		struct expression *initializer = get_symbol_initializer(sym);
+		if (initializer) {
+			struct symbol *node_type = base_type->ctype.base_type;
+			int count = count_array_initializer(S, node_type, initializer);
+
+			if (node_type && node_type->bit_size >= 0)
+				bit_size = (int) dmrC_array_element_offset(S->C->target, node_type->bit_size, count);
+			/* Note that the bit_size will be set on parent SYM_NODE rather than here */
+			//base_type->bit_size = bit_size;
+		}
+	}
+	
+	sym->bit_size = bit_size;
+	return sym;
+}
+
+static struct symbol *examine_enum_type(struct global_symbols_t *S, struct symbol *sym)
+{
+	struct symbol *base_type = examine_base_type(S, sym);
+
+	sym->ctype.modifiers |= (base_type->ctype.modifiers & MOD_SIGNEDNESS);
+	sym->bit_size = S->C->target->bits_in_enum;
+	if (base_type->bit_size > sym->bit_size)
+		sym->bit_size = base_type->bit_size;
+	sym->ctype.alignment = S->C->target->enum_alignment;
+	if (base_type->ctype.alignment > sym->ctype.alignment)
+		sym->ctype.alignment = base_type->ctype.alignment;
+	return sym;
+}
+
+static struct symbol *examine_pointer_type(struct global_symbols_t *S, struct symbol *sym)
+{
+	/*
+	 * We need to set the pointer size first, and
+	 * examine the thing we point to only afterwards.
+	 * That's because this pointer type may end up
+	 * being needed for the base type size evaluation.
+	 */
+	if (!sym->bit_size)
+		sym->bit_size = S->C->target->bits_in_pointer;
+	if (!sym->ctype.alignment)
+		sym->ctype.alignment = S->C->target->pointer_alignment;
+	return sym;
+}
+
+/*
+ * Fill in type size and alignment information for
+ * regular SYM_TYPE things.
+ */
+struct symbol *dmrC_examine_symbol_type(struct global_symbols_t *S, struct symbol * sym)
+{
+	if (!sym)
+		return sym;
+
+	/* Already done? */
+	if (sym->examined)
+		return sym;
+	sym->examined = 1;
+
+	switch (sym->type) {
+	case SYM_FN:
+	case SYM_NODE:
+		return examine_node_type(S, sym);
+	case SYM_ARRAY:
+		return examine_array_type(S, sym);
+	case SYM_STRUCT:
+		return examine_struct_union_type(S, sym, 1);
+	case SYM_UNION:
+		return examine_struct_union_type(S, sym, 0);
+	case SYM_PTR:
+		return examine_pointer_type(S, sym);
+	case SYM_ENUM:
+		return examine_enum_type(S, sym);
+	case SYM_BITFIELD:
+		return examine_bitfield_type(S, sym);
+	case SYM_BASETYPE:
+		/* Size and alignment had better already be set up */
+		return sym;
+	case SYM_TYPEOF: {
+		struct symbol *base = dmrC_evaluate_expression(S->C, sym->initializer);
+		if (base) {
+			unsigned long mod = 0;
+
+			if (dmrC_is_bitfield_type(base))
+				dmrC_warning(S->C, base->pos, "typeof applied to bitfield type");
+			if (base->type == SYM_NODE) {
+				mod |= base->ctype.modifiers & MOD_TYPEOF;
+				base = base->ctype.base_type;
+			}
+			sym->type = SYM_NODE;
+			sym->ctype.modifiers = mod;
+			sym->ctype.base_type = base;
+			return examine_node_type(S, sym);
+		}
+		break;
+	}
+	case SYM_PREPROCESSOR:
+		dmrC_sparse_error(S->C, sym->pos, "ctype on preprocessor command? (%s)", dmrC_show_ident(S->C, sym->ident));
+		return NULL;
+	case SYM_UNINITIALIZED:
+		dmrC_sparse_error(S->C, sym->pos, "ctype on uninitialized symbol %p", sym);
+		return NULL;
+	case SYM_RESTRICT:
+		examine_base_type(S, sym);
+		return sym;
+	case SYM_FOULED:
+		examine_base_type(S, sym);
+		return sym;
+	default:
+		dmrC_sparse_error(S->C, sym->pos, "Examining unknown symbol type %d", sym->type);
+		break;
+	}
+	return sym;
+}
+
+const char* dmrC_get_type_name(enum type type)
+{
+	const char *type_lookup[] = {
+	[SYM_UNINITIALIZED] = "uninitialized",
+	[SYM_PREPROCESSOR] = "preprocessor",
+	[SYM_BASETYPE] = "basetype",
+	[SYM_NODE] = "node",
+	[SYM_PTR] = "pointer",
+	[SYM_FN] = "function",
+	[SYM_ARRAY] = "array",
+	[SYM_STRUCT] = "struct",
+	[SYM_UNION] = "union",
+	[SYM_ENUM] = "enum",
+	[SYM_TYPEDEF] = "typedef",
+	[SYM_TYPEOF] = "typeof",
+	[SYM_MEMBER] = "member",
+	[SYM_BITFIELD] = "bitfield",
+	[SYM_LABEL] = "label",
+	[SYM_RESTRICT] = "restrict",
+	[SYM_FOULED] = "fouled",
+	[SYM_KEYWORD] = "keyword",
+	[SYM_BAD] = "bad"};
+
+	if (type <= SYM_BAD)
+		return type_lookup[type];
+	else
+		return NULL;
+}
+
+struct symbol *dmrC_examine_pointer_target(struct global_symbols_t *S, struct symbol *sym)
+{
+	return examine_base_type(S, sym);
+}
+
+void dmrC_create_fouled(struct global_symbols_t *S, struct symbol *type)
+{
+	if (type->bit_size < S->C->target->bits_in_int) {
+		struct symbol *news = dmrC_alloc_symbol(S, type->pos, type->type);
+		*news = *type;
+		news->bit_size = S->C->target->bits_in_int;
+		news->type = SYM_FOULED;
+		news->ctype.base_type = type;
+		dmrC_add_symbol(S->C, &S->restr, type);
+		dmrC_add_symbol(S->C, &S->fouled, news);
+	}
+}
+
+struct symbol *dmrC_befoul(struct global_symbols_t *S, struct symbol *type)
+{
+	struct symbol *t1, *t2;
+	while (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	PREPARE_PTR_LIST(S->restr, t1);
+	PREPARE_PTR_LIST(S->fouled, t2);
+	for (;;) {
+		if (t1 == type)
+			return t2;
+		if (!t1)
+			break;
+		NEXT_PTR_LIST(t1);
+		NEXT_PTR_LIST(t2);
+	}
+	FINISH_PTR_LIST(t2);
+	FINISH_PTR_LIST(t1);
+	return NULL;
+}
+
+void dmrC_check_declaration(struct global_symbols_t *S, struct symbol *sym)
+{
+	int warned = 0;
+	struct symbol *next = sym;
+
+	while ((next = next->next_id) != NULL) {
+		if (next->ns != sym->ns)
+			continue;
+		if (sym->scope == next->scope) {
+			sym->same_symbol = next;
+			return;
+		}
+		/* Extern in block level matches a TOPLEVEL non-static symbol */
+		if (sym->ctype.modifiers & MOD_EXTERN) {
+			if ((next->ctype.modifiers & (MOD_TOPLEVEL|MOD_STATIC)) == MOD_TOPLEVEL) {
+				sym->same_symbol = next;
+				return;
+			}
+		}
+
+
+		if (!S->C->Wshadow || warned)
+			continue;
+		if (dmrC_get_sym_type(next) == SYM_FN)
+			continue;
+		warned = 1;
+		dmrC_warning(S->C, sym->pos, "symbol '%s' shadows an earlier one", dmrC_show_ident(S->C, sym->ident));
+		dmrC_info(S->C, next->pos, "originally declared here");
+	}
+}
+
+void dmrC_bind_symbol(struct global_symbols_t *S, struct symbol *sym, struct ident *ident, enum namespace_type ns)
+{
+	struct scope *scope;
+	if (sym->bound) {
+		dmrC_sparse_error(S->C, sym->pos, "internal error: symbol type already bound");
+		return;
+	}
+	if (ident->reserved && (ns & (NS_TYPEDEF | NS_STRUCT | NS_LABEL | NS_SYMBOL))) {
+		dmrC_sparse_error(S->C, sym->pos, "Trying to use reserved word '%s' as identifier", dmrC_show_ident(S->C, ident));
+		return;
+	}
+	sym->ns = ns;
+	sym->next_id = ident->symbols;
+	ident->symbols = sym;
+	if (sym->ident && sym->ident != ident)
+		dmrC_warning(S->C, sym->pos, "Symbol '%s' already bound", dmrC_show_ident(S->C, sym->ident));
+	sym->ident = ident;
+	sym->bound = 1;
+
+	scope = S->C->block_scope;
+	if (ns == NS_SYMBOL && dmrC_toplevel(S->C, scope)) {
+		unsigned mod = MOD_ADDRESSABLE | MOD_TOPLEVEL;
+
+		scope = S->C->global_scope;
+		if (sym->ctype.modifiers & MOD_STATIC ||
+		    dmrC_is_extern_inline(sym)) {
+			scope = S->C->file_scope;
+			mod = MOD_TOPLEVEL;
+		}
+		sym->ctype.modifiers |= mod;
+	}
+	if (ns == NS_MACRO)
+		scope = S->C->file_scope;
+	if (ns == NS_LABEL)
+		scope = S->C->function_scope;
+	dmrC_bind_scope(S->C, sym, scope);
+}
+
+struct symbol *dmrC_create_symbol(struct global_symbols_t *S, int stream, const char *name, int type, int ns)
+{
+	struct ident *ident = dmrC_built_in_ident(S->C, name);
+	struct symbol *sym = dmrC_lookup_symbol(ident, ns);
+
+	if (sym && sym->type != type)
+		dmrC_die(S->C, "symbol %s created with different types: %d old %d", name,
+				type, sym->type);
+
+	if (!sym) {
+		struct token *token = dmrC_built_in_token(S->C, stream, ident);
+
+		sym = dmrC_alloc_symbol(S, token->pos, type);
+		dmrC_bind_symbol(S, sym, token->ident, ns);
+	}
+	return sym;
+}
+
+void dmrC_init_symbols(struct dmr_C *C)
+{
+	struct global_symbols_t *S = (struct global_symbols_t *) calloc(1, sizeof(struct global_symbols_t));
+	C->S = S;
+	S->C = C; 
+	dmrC_allocator_init(&S->context_allocator, "contexts", sizeof(struct context), __alignof__(struct context),
+		CHUNK);
+	dmrC_allocator_init(&S->global_ident_allocator, "global_identifiers", sizeof(struct ident),
+		__alignof__(struct ident), CHUNK);
+	dmrC_allocator_init(&S->symbol_allocator, "symbols", sizeof(struct symbol),
+		__alignof__(struct symbol), CHUNK);
+
+	int stream = dmrC_init_stream(C, "builtin", -1, C->T->includepath);
+
+#define __INIT_IDENT(n, str, res) (struct ident *) dmrC_allocator_allocate(&S->global_ident_allocator, sizeof(str)); S->n->len = sizeof(str)-1; memcpy(S->n->name, str, sizeof(str)); S->n->reserved = res;
+#define __IDENT(n,str,res) \
+	{S->n  = __INIT_IDENT(n, str, res)}
+#include "ident-list.h"
+
+#define __IDENT(n,str,res) \
+	dmrC_hash_ident(C, S->n)
+#include "ident-list.h"
+
+	dmrC_init_parser(C, stream);
+	dmrC_init_builtins(C, stream);
+}
+
+void dmrC_destroy_symbols(struct dmr_C *C) {
+	/* tokenizer must be destroyed before this */
+	assert(C->T == NULL);
+
+	dmrC_destroy_parser(C);
+	assert(C->P == NULL);
+
+	struct global_symbols_t *S = C->S;
+
+	dmrC_allocator_destroy(&S->context_allocator);
+	dmrC_allocator_destroy(&S->global_ident_allocator);
+	dmrC_allocator_destroy(&S->symbol_allocator);
+
+	free(S);
+	C->S = NULL;
+}
+
+void dmrC_init_ctype(struct dmr_C *C)
+{
+	assert(C);
+	struct global_symbols_t *S = C->S;
+	assert(S);
+	struct target_t *T = C->target;
+	assert(T);
+	const struct ctype_declare *ctype;
+
+#ifdef _MSC_VER
+	if (sizeof(long long) == sizeof(size_t)) {
+		T->size_t_ctype = &S->ullong_ctype;
+		T->ssize_t_ctype = &S->llong_ctype;
+	}
+	else {
+		assert(sizeof(int) == sizeof(size_t));
+		T->size_t_ctype = &S->uint_ctype;
+		T->ssize_t_ctype = &S->int_ctype;
+	}
+#else
+	T->size_t_ctype = &S->uint_ctype;
+	T->ssize_t_ctype = &S->int_ctype;
+#endif
+
+#define MOD_ESIGNED (MOD_SIGNED | MOD_EXPLICITLY_SIGNED)
+#define MOD_LL (MOD_LONG | MOD_LONGLONG)
+#define MOD_LLL MOD_LONGLONGLONG
+	const struct ctype_declare {
+		struct symbol *ptr;
+		enum type type;
+		unsigned long modifiers;
+		int *bit_size;
+		int *maxalign;
+		struct symbol *base_type;
+	} ctype_declaration[] = {
+		{ &S->bool_ctype,	    SYM_BASETYPE, MOD_UNSIGNED,		    &T->bits_in_bool,	     &T->max_int_alignment, &S->int_type },
+		{ &S->void_ctype,	    SYM_BASETYPE, 0,			    NULL,	     NULL,		 NULL },
+		{ &S->type_ctype,	    SYM_BASETYPE, MOD_TYPE,		    NULL,		     NULL,		 NULL },
+		{ &S->incomplete_ctype,SYM_BASETYPE, 0,			    NULL,		     NULL,		 NULL },
+		{ &S->bad_ctype,	    SYM_BASETYPE, 0,			    NULL,		     NULL,		 NULL },
+
+		{ &S->char_ctype,	    SYM_BASETYPE, MOD_SIGNED | MOD_CHAR,    &T->bits_in_char,	     &T->max_int_alignment, &S->int_type },
+		{ &S->schar_ctype,	    SYM_BASETYPE, MOD_ESIGNED | MOD_CHAR,   &T->bits_in_char,	     &T->max_int_alignment, &S->int_type },
+		{ &S->uchar_ctype,	    SYM_BASETYPE, MOD_UNSIGNED | MOD_CHAR,  &T->bits_in_char,	     &T->max_int_alignment, &S->int_type },
+		{ &S->short_ctype,	    SYM_BASETYPE, MOD_SIGNED | MOD_SHORT,   &T->bits_in_short,	     &T->max_int_alignment, &S->int_type },
+		{ &S->sshort_ctype,    SYM_BASETYPE, MOD_ESIGNED | MOD_SHORT,  &T->bits_in_short,	     &T->max_int_alignment, &S->int_type },
+		{ &S->ushort_ctype,    SYM_BASETYPE, MOD_UNSIGNED | MOD_SHORT, &T->bits_in_short,	     &T->max_int_alignment, &S->int_type },
+		{ &S->int_ctype,	    SYM_BASETYPE, MOD_SIGNED,		    &T->bits_in_int,	     &T->max_int_alignment, &S->int_type },
+		{ &S->sint_ctype,	    SYM_BASETYPE, MOD_ESIGNED,		    &T->bits_in_int,	     &T->max_int_alignment, &S->int_type },
+		{ &S->uint_ctype,	    SYM_BASETYPE, MOD_UNSIGNED,		    &T->bits_in_int,	     &T->max_int_alignment, &S->int_type },
+		{ &S->long_ctype,	    SYM_BASETYPE, MOD_SIGNED | MOD_LONG,    &T->bits_in_long,	     &T->max_int_alignment, &S->int_type },
+		{ &S->slong_ctype,	    SYM_BASETYPE, MOD_ESIGNED | MOD_LONG,   &T->bits_in_long,	     &T->max_int_alignment, &S->int_type },
+		{ &S->ulong_ctype,	    SYM_BASETYPE, MOD_UNSIGNED | MOD_LONG,  &T->bits_in_long,	     &T->max_int_alignment, &S->int_type },
+		{ &S->llong_ctype,	    SYM_BASETYPE, MOD_SIGNED | MOD_LL,	    &T->bits_in_longlong,       &T->max_int_alignment, &S->int_type },
+		{ &S->sllong_ctype,    SYM_BASETYPE, MOD_ESIGNED | MOD_LL,	    &T->bits_in_longlong,       &T->max_int_alignment, &S->int_type },
+		{ &S->ullong_ctype,    SYM_BASETYPE, MOD_UNSIGNED | MOD_LL,    &T->bits_in_longlong,       &T->max_int_alignment, &S->int_type },
+		{ &S->lllong_ctype,    SYM_BASETYPE, MOD_SIGNED | MOD_LLL,	    &T->bits_in_longlonglong,   &T->max_int_alignment, &S->int_type },
+		{ &S->slllong_ctype,   SYM_BASETYPE, MOD_ESIGNED | MOD_LLL,    &T->bits_in_longlonglong,   &T->max_int_alignment, &S->int_type },
+		{ &S->ulllong_ctype,   SYM_BASETYPE, MOD_UNSIGNED | MOD_LLL,   &T->bits_in_longlonglong,   &T->max_int_alignment, &S->int_type },
+
+		{ &S->float_ctype,	    SYM_BASETYPE,  0,			    &T->bits_in_float,          &T->max_fp_alignment,  &S->fp_type },
+		{ &S->double_ctype,    SYM_BASETYPE, MOD_LONG,		    &T->bits_in_double,         &T->max_fp_alignment,  &S->fp_type },
+		{ &S->ldouble_ctype,   SYM_BASETYPE, MOD_LONG | MOD_LONGLONG,  &T->bits_in_longdouble,     &T->max_fp_alignment,  &S->fp_type },
+
+		{ &S->string_ctype,    SYM_PTR,	  0,			    &T->bits_in_pointer,        &T->pointer_alignment, &S->char_ctype },
+		{ &S->ptr_ctype,	    SYM_PTR,	  0,			    &T->bits_in_pointer,        &T->pointer_alignment, &S->void_ctype },
+		{ &S->null_ctype,	    SYM_PTR,	  0,			    &T->bits_in_pointer,        &T->pointer_alignment, &S->void_ctype },
+		{ &S->label_ctype,	    SYM_PTR,	  0,			    &T->bits_in_pointer,        &T->pointer_alignment, &S->void_ctype },
+		{ &S->lazy_ptr_ctype,  SYM_PTR,	  0,			    &T->bits_in_pointer,        &T->pointer_alignment, &S->void_ctype },
+		{ NULL, SYM_UNINITIALIZED, 0, NULL, NULL, NULL }
+	};
+#undef MOD_LLL
+#undef MOD_LL
+#undef MOD_ESIGNED
+
+
+	for (ctype = ctype_declaration ; ctype->ptr; ctype++) {
+		struct symbol *sym = ctype->ptr;
+		unsigned long bit_size = ctype->bit_size ? *ctype->bit_size : -1;
+		unsigned long maxalign = ctype->maxalign ? *ctype->maxalign : 0;
+		unsigned long alignment = dmrC_bits_to_bytes(T, bit_size);
+
+		if (alignment > maxalign)
+			alignment = maxalign;
+		sym->type = ctype->type;
+		sym->bit_size = bit_size;
+		sym->ctype.alignment = alignment;
+		sym->ctype.base_type = ctype->base_type;
+		sym->ctype.modifiers = ctype->modifiers;
+	}
+
+	S->typenames[0].sym = &S->char_ctype; S->typenames[0].name = "char";
+	S->typenames[1].sym = &S->schar_ctype; S->typenames[1].name = "signed char";
+	S->typenames[2].sym = &S->uchar_ctype; S->typenames[2].name = "unsigned char";
+	S->typenames[3].sym = &S->short_ctype; S->typenames[3].name = "short";
+	S->typenames[4].sym = &S->sshort_ctype; S->typenames[4].name = "signed short";
+	S->typenames[5].sym = &S->ushort_ctype; S->typenames[5].name = "unsigned short";
+	S->typenames[6].sym = &S->int_ctype; S->typenames[6].name = "int";
+	S->typenames[7].sym = &S->sint_ctype; S->typenames[7].name = "signed int";
+	S->typenames[8].sym = &S->uint_ctype; S->typenames[8].name = "unsigned int";
+	S->typenames[9].sym = &S->slong_ctype; S->typenames[9].name = "signed long";
+	S->typenames[10].sym = &S->long_ctype; S->typenames[10].name = "long";
+	S->typenames[11].sym = &S->ulong_ctype; S->typenames[11].name = "unsigned long";
+	S->typenames[12].sym = &S->llong_ctype; S->typenames[12].name = "long long";
+	S->typenames[13].sym = &S->sllong_ctype; S->typenames[13].name = "signed long long";
+	S->typenames[14].sym = &S->ullong_ctype; S->typenames[14].name = "unsigned long long";
+	S->typenames[15].sym = &S->lllong_ctype; S->typenames[15].name = "long long long";
+	S->typenames[16].sym = &S->slllong_ctype; S->typenames[16].name = "signed long long long";
+	S->typenames[17].sym = &S->ulllong_ctype; S->typenames[17].name = "unsigned long long long";
+
+	S->typenames[18].sym = &S->void_ctype; S->typenames[18].name = "void";
+	S->typenames[19].sym = &S->bool_ctype; S->typenames[19].name = "bool";
+	S->typenames[20].sym = &S->string_ctype; S->typenames[20].name = "string";
+
+	S->typenames[21].sym = &S->float_ctype; S->typenames[21].name = "float";
+	S->typenames[22].sym = &S->double_ctype; S->typenames[22].name = "double";
+	S->typenames[23].sym = &S->ldouble_ctype; S->typenames[23].name = "long double";
+	S->typenames[24].sym = &S->incomplete_ctype; S->typenames[24].name = "incomplete type";
+	S->typenames[25].sym = &S->int_type; S->typenames[25].name = "abstract int";
+	S->typenames[26].sym = &S->fp_type; S->typenames[26].name = "abstract fp";
+	S->typenames[27].sym = &S->label_ctype; S->typenames[27].name = "label type";
+	S->typenames[28].sym = &S->bad_ctype; S->typenames[28].name = "bad type";
+	S->typenames[29].sym = NULL, S->typenames[29].name = NULL;
+}
+

dmr_c/src/symbol.h

@@ -0,0 +1,595 @@
+#ifndef DMR_C_SYMBOL_H
+#define DMR_C_SYMBOL_H
+
+/*
+* Basic symbol and namespace definitions.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <target.h>
+#include <token.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*
+* An identifier with semantic meaning is a "symbol".
+*
+* There's a 1:n relationship: each symbol is always
+* associated with one identifier, while each identifier
+* can have one or more semantic meanings due to C scope
+* rules.
+*
+* The progression is symbol -> token -> identifier. The
+* token contains the information on where the symbol was
+* declared.
+*/
+enum namespace_type {
+	NS_NONE = 0,
+	NS_MACRO = 1,
+	NS_TYPEDEF = 2,
+	NS_STRUCT = 4,  // Also used for unions and enums.
+	NS_LABEL = 8,
+	NS_SYMBOL = 16,
+	NS_ITERATOR = 32,
+	NS_PREPROCESSOR = 64,
+	NS_UNDEF = 128,
+	NS_KEYWORD = 256,
+};
+
+enum type {
+	SYM_UNINITIALIZED,
+	SYM_PREPROCESSOR,
+	SYM_BASETYPE,
+	SYM_NODE,
+	SYM_PTR,
+	SYM_FN,
+	SYM_ARRAY,
+	SYM_STRUCT,
+	SYM_UNION,
+	SYM_ENUM,
+	SYM_TYPEDEF,
+	SYM_TYPEOF,
+	SYM_MEMBER,
+	SYM_BITFIELD,
+	SYM_LABEL,
+	SYM_RESTRICT,
+	SYM_FOULED,
+	SYM_KEYWORD,
+	SYM_BAD,
+};
+
+enum keyword {
+	KW_SPECIFIER 	= 1 << 0,
+	KW_MODIFIER	= 1 << 1,
+	KW_QUALIFIER	= 1 << 2,
+	KW_ATTRIBUTE	= 1 << 3,
+	KW_STATEMENT	= 1 << 4,
+	KW_ASM		= 1 << 5,
+	KW_MODE		= 1 << 6,
+	KW_SHORT	= 1 << 7,
+	KW_LONG		= 1 << 8,
+	KW_EXACT	= 1 << 9,
+};
+
+struct context {
+	struct expression *context_expr;
+	unsigned int in, out;
+};
+
+DECLARE_PTR_LIST(context_list, struct context);
+struct phi_map;
+
+struct ctype {
+	unsigned long modifiers;
+	unsigned long alignment;
+	struct context_list *contexts;
+	unsigned int as;
+	struct symbol *base_type;
+};
+
+struct decl_state {
+	struct ctype ctype;
+	struct ident **ident;
+	struct symbol_op *mode;
+	unsigned char prefer_abstract, is_inline, storage_class, is_tls;
+};
+
+struct symbol;
+DECLARE_PTR_LIST(symbol_list, struct symbol);
+
+struct symbol_op {
+	enum keyword type;
+	int (*evaluate)(struct dmr_C *, struct expression *);
+	int (*expand)(struct dmr_C *, struct expression *, int);
+	int (*args)(struct dmr_C *, struct expression *);
+
+	/* keywords */
+	struct token *(*declarator)(struct dmr_C *, struct token *token,
+				    struct decl_state *ctx);
+	struct token *(*statement)(struct dmr_C *, struct token *token, struct statement *stmt);
+	struct token *(*toplevel)(struct dmr_C *, struct token *token, struct symbol_list **list);
+	struct token *(*attribute)(struct dmr_C *, struct token *token, struct symbol *attr,
+				   struct decl_state *ctx);
+	struct symbol *(*to_mode)(struct dmr_C *, struct symbol *);
+
+	int test, set, cls;
+};
+
+#define SYM_ATTR_WEAK 0
+#define SYM_ATTR_NORMAL 1
+#define SYM_ATTR_STRONG 2
+
+struct symbol {
+	enum type type : 8;
+	enum namespace_type ns : 9;
+	unsigned char used : 1, attr : 2, enum_member : 1, bound : 1;
+	struct position pos;    /* Where this symbol was declared */
+	struct position endpos; /* Where this symbol ends*/
+	struct ident *ident;    /* What identifier this symbol is associated with */
+	struct symbol *next_id; /* Next semantic symbol that shares this identifier */
+	struct symbol *replace; /* What is this symbol shadowed by in copy-expression */
+	struct scope *scope;
+	union {
+		struct symbol	*same_symbol;
+		struct symbol	*next_subobject;
+	};
+
+	struct symbol_op *op;
+
+	union {
+		struct /* NS_MACRO */ {
+			struct token *expansion;
+			struct token *arglist;
+			struct scope *used_in;
+		};
+		struct /* NS_PREPROCESSOR */ {
+			int (*handler)(struct dmr_C *, struct stream *, struct token **, struct token *);
+			int normal;
+		};
+		struct /* NS_SYMBOL */ {
+			unsigned long	offset;
+			int		bit_size;
+			unsigned int	bit_offset:8,
+					arg_count:10,
+					variadic:1,
+					initialized:1,
+					examined:1,
+					expanding:1,
+					evaluated:1,
+					string:1,
+					designated_init:1,
+					forced_arg:1,
+					transparent_union:1;
+			struct expression *array_size;
+			struct ctype ctype;
+			struct symbol_list *arguments;
+			struct statement *stmt;
+			struct symbol_list *symbol_list;
+			struct statement *inline_stmt;
+			struct symbol_list *inline_symbol_list;
+			struct expression *initializer;
+			struct entrypoint *ep;
+			long long value;		/* Initial value */
+			struct symbol *definition;
+		};
+	};
+	union /* backend */ {
+		struct basic_block *bb_target;	/* label */
+		void *aux;			/* Auxiliary info, e.g. backend information */
+		struct {			/* sparse ctags */
+			char kind;
+			unsigned char visited:1;
+		};
+	};
+	pseudo_t pseudo;
+	DMRC_BACKEND_TYPE priv;
+};
+
+/* Modifiers */
+#define MOD_AUTO	0x0001
+#define MOD_REGISTER	0x0002
+#define MOD_STATIC	0x0004
+#define MOD_EXTERN	0x0008
+
+#define MOD_CONST	0x0010
+#define MOD_VOLATILE	0x0020
+#define MOD_SIGNED	0x0040
+#define MOD_UNSIGNED	0x0080
+
+#define MOD_CHAR	0x0100
+#define MOD_SHORT	0x0200
+#define MOD_LONG	0x0400
+#define MOD_LONGLONG	0x0800
+#define MOD_LONGLONGLONG	0x1000
+#define MOD_PURE	0x2000
+
+#define MOD_TYPEDEF	0x10000
+
+#define MOD_TLS		0x20000
+#define MOD_INLINE	0x40000
+#define MOD_ADDRESSABLE	0x80000
+
+#define MOD_NOCAST	0x100000
+#define MOD_NODEREF	0x200000
+#define MOD_ACCESSED	0x400000
+#define MOD_TOPLEVEL	0x800000	// scoping..
+
+#define MOD_ASSIGNED	0x2000000
+#define MOD_TYPE	0x4000000
+#define MOD_SAFE	0x8000000	// non-null/non-trapping pointer
+
+#define MOD_USERTYPE	0x10000000
+#define MOD_NORETURN	0x20000000
+#define MOD_EXPLICITLY_SIGNED	0x40000000
+#define MOD_BITWISE	0x80000000
+
+
+#define MOD_NONLOCAL	(MOD_EXTERN | MOD_TOPLEVEL)
+#define MOD_STORAGE	(MOD_AUTO | MOD_REGISTER | MOD_STATIC | MOD_EXTERN | MOD_INLINE | MOD_TOPLEVEL)
+#define MOD_SIGNEDNESS	(MOD_SIGNED | MOD_UNSIGNED | MOD_EXPLICITLY_SIGNED)
+#define MOD_LONG_ALL	(MOD_LONG | MOD_LONGLONG | MOD_LONGLONGLONG)
+#define MOD_SPECIFIER	(MOD_CHAR | MOD_SHORT | MOD_LONG_ALL | MOD_SIGNEDNESS)
+#define MOD_SIZE	(MOD_CHAR | MOD_SHORT | MOD_LONG_ALL)
+#define MOD_IGNORE (MOD_TOPLEVEL | MOD_STORAGE | MOD_ADDRESSABLE |	\
+	MOD_ASSIGNED | MOD_USERTYPE | MOD_ACCESSED | MOD_EXPLICITLY_SIGNED)
+#define MOD_PTRINHERIT (MOD_VOLATILE | MOD_CONST | MOD_NODEREF | MOD_NORETURN | MOD_NOCAST)
+/* modifiers preserved by typeof() operator */
+#define MOD_TYPEOF	(MOD_VOLATILE | MOD_CONST | MOD_NOCAST | MOD_SPECIFIER)
+
+struct ctype_name {
+	struct symbol *sym;
+	const char *name;
+};
+
+struct global_symbols_t {
+	struct dmr_C *C;
+
+	/* Abstract types */
+	struct symbol	int_type,
+			fp_type;
+
+	/* C types */
+	struct symbol bool_ctype, void_ctype, type_ctype,
+			char_ctype, schar_ctype, uchar_ctype,
+			short_ctype, sshort_ctype, ushort_ctype,
+			int_ctype, sint_ctype, uint_ctype,
+			long_ctype, slong_ctype, ulong_ctype,
+			llong_ctype, sllong_ctype, ullong_ctype,
+			lllong_ctype, slllong_ctype, ulllong_ctype,
+			float_ctype, double_ctype, ldouble_ctype,
+			string_ctype, ptr_ctype, lazy_ptr_ctype,
+			incomplete_ctype, label_ctype, bad_ctype,
+			null_ctype;
+
+	/* Special internal symbols */
+	struct symbol zero_int;
+
+	/*
+	* Secondary symbol list for stuff that needs to be output because it
+	* was used.
+	*/
+	struct symbol_list *translation_unit_used_list;
+
+	struct allocator context_allocator;
+	struct allocator symbol_allocator;
+	struct allocator global_ident_allocator;
+
+	struct symbol_list *restr, *fouled;
+
+	struct ctype_name typenames[30];
+
+#define __IDENT(n, str, res) struct ident *n
+#include "ident-list.h"
+#undef __IDENT
+};
+
+extern void dmrC_init_symbols(struct dmr_C *C);
+extern void dmrC_init_ctype(struct dmr_C *C);
+extern void dmrC_init_builtins(struct dmr_C *C, int stream);
+extern void dmrC_destroy_symbols(struct dmr_C *C);
+
+extern struct context *dmrC_alloc_context(struct global_symbols_t *S);
+extern void dmrC_access_symbol(struct global_symbols_t *S, struct symbol *sym);
+
+extern const char *dmrC_type_difference(struct dmr_C *C, struct ctype *c1, struct ctype *c2,
+				   unsigned long mod1, unsigned long mod2);
+
+extern struct symbol *dmrC_lookup_symbol(struct ident *, enum namespace_type);
+extern struct symbol *dmrC_create_symbol(struct global_symbols_t *S, int stream, const char *name, int type, int ns);
+
+extern struct symbol *dmrC_alloc_symbol(struct global_symbols_t *S,
+				   struct position pos, int type);
+extern void dmrC_show_type(struct dmr_C *C, struct symbol *);
+extern const char *dmrC_modifier_string(struct dmr_C *C, unsigned long mod);
+extern void dmrC_show_symbol(struct dmr_C *C, struct symbol *);
+extern int dmrC_show_symbol_expr_init(struct dmr_C *C, struct symbol *sym);
+extern void dmrC_show_symbol_list(struct dmr_C *C, struct symbol_list *, const char *);
+extern void dmrC_bind_symbol(struct global_symbols_t *S, struct symbol *sym,
+			struct ident *ident, enum namespace_type ns);
+
+extern struct symbol *dmrC_examine_symbol_type(struct global_symbols_t *S,
+					  struct symbol *sym);
+extern struct symbol *dmrC_examine_pointer_target(struct global_symbols_t *S,
+					     struct symbol *sym);
+extern const char *dmrC_show_typename(struct dmr_C *C, struct symbol *sym);
+extern const char *dmrC_builtin_typename(struct dmr_C *C, struct symbol *sym);
+extern const char *dmrC_builtin_ctypename(struct dmr_C *C, struct ctype *ctype);
+extern const char *dmrC_get_type_name(enum type type);
+
+extern void dmrC_debug_symbol(struct dmr_C *C, struct symbol *);
+extern void dmrC_merge_type(struct symbol *sym, struct symbol *base_type);
+extern void dmrC_check_declaration(struct global_symbols_t *S, struct symbol *sym);
+
+static inline struct symbol *dmrC_get_base_type(struct global_symbols_t *S,
+					   const struct symbol *sym)
+{
+	return dmrC_examine_symbol_type(S, sym->ctype.base_type);
+}
+
+static inline int dmrC_is_int_type(struct global_symbols_t *S,
+			      const struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	if (type->type == SYM_ENUM)
+		type = type->ctype.base_type;
+	return type->type == SYM_BITFIELD ||
+	       type->ctype.base_type == &S->int_type;
+}
+
+static inline int dmrC_is_enum_type(const struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return (type->type == SYM_ENUM);
+}
+
+static inline int dmrC_is_type_type(struct symbol *type)
+{
+	return (type->ctype.modifiers & MOD_TYPE) != 0;
+}
+
+static inline int dmrC_is_ptr_type(struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return type->type == SYM_PTR || type->type == SYM_ARRAY || type->type == SYM_FN;
+}
+
+static inline int dmrC_is_func_type(struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return type->type == SYM_FN;
+}
+
+static inline int dmrC_is_array_type(struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return type->type == SYM_ARRAY;
+}
+
+static inline int dmrC_is_float_type(struct global_symbols_t *S, struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return type->ctype.base_type == &S->fp_type;
+}
+
+static inline int dmrC_is_byte_type(const struct target_t *target,
+			       struct symbol *type)
+{
+	return type->bit_size == target->bits_in_char && type->type != SYM_BITFIELD;
+}
+
+static inline int dmrC_is_void_type(struct global_symbols_t *S, struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return type == &S->void_ctype;
+}
+
+static inline int dmrC_is_bool_type(struct global_symbols_t *S, struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	return type == &S->bool_ctype;
+}
+
+static inline int dmrC_is_scalar_type(struct global_symbols_t *S, struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	switch (type->type) {
+	case SYM_ENUM:
+	case SYM_BITFIELD:
+	case SYM_PTR:
+	case SYM_ARRAY:		// OK, will be a PTR after conversion
+	case SYM_FN:
+	case SYM_RESTRICT:	// OK, always integer types
+		return 1;
+	default:
+		break;
+	}
+	if (type->ctype.base_type == &S->int_type)
+		return 1;
+	if (type->ctype.base_type == &S->fp_type)
+		return 1;
+	return 0;
+}
+
+// From Luc - sssa-mini
+static inline int dmrC_is_simple_type(struct global_symbols_t *S, struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	switch (type->type) {
+	case SYM_ENUM:
+	case SYM_BITFIELD:
+	case SYM_PTR:
+	case SYM_RESTRICT:	// OK, always integer types
+		return 1;
+	// Following is causing failures because the IR
+	// attempts to store values into unions or structs
+	//case SYM_STRUCT:
+	//case SYM_UNION:
+	//	return type->bit_size <= S->long_ctype.bit_size;
+	default:
+		break;
+	}
+	if (type->ctype.base_type == &S->int_type)
+		return 1;
+	if (type->ctype.base_type == &S->fp_type)
+		return 1;
+	return 0;
+}
+
+// From Luc - sssa-mini
+static inline int dmrC_is_simple_var(struct global_symbols_t *S, struct symbol *var)
+{
+	if (!dmrC_is_simple_type(S, var))
+		return 0;
+#define	MOD_NONREG	(MOD_STATIC|MOD_NONLOCAL|MOD_ADDRESSABLE|MOD_VOLATILE)
+	if (var->ctype.modifiers & MOD_NONREG)
+		return 0;
+	return 1;
+}
+
+static inline int dmrC_is_function(struct symbol *type)
+{
+	return type && type->type == SYM_FN;
+}
+
+static inline int dmrC_is_extern_inline(struct symbol *sym)
+{
+	return (sym->ctype.modifiers & MOD_EXTERN) &&
+		(sym->ctype.modifiers & MOD_INLINE) &&
+		dmrC_is_function(sym->ctype.base_type);
+}
+
+static inline int dmrC_is_toplevel(struct symbol *sym)
+{
+	return (sym->ctype.modifiers & MOD_TOPLEVEL);
+}
+
+static inline int dmrC_is_extern(struct symbol *sym)
+{
+	return (sym->ctype.modifiers & MOD_EXTERN);
+}
+
+static inline int dmrC_is_static(struct symbol *sym)
+{
+	return (sym->ctype.modifiers & MOD_STATIC);
+}
+
+static int dmrC_is_signed_type(struct symbol *sym)
+{
+	if (sym->type == SYM_NODE)
+		sym = sym->ctype.base_type;
+	if (sym->type == SYM_PTR)
+		return 0;
+	return !(sym->ctype.modifiers & MOD_UNSIGNED);
+}
+
+static inline int dmrC_is_unsigned(struct symbol *sym)
+{
+	return !dmrC_is_signed_type(sym);
+}
+
+static inline int dmrC_get_sym_type(struct symbol *type)
+{
+	if (type->type == SYM_NODE)
+		type = type->ctype.base_type;
+	if (type->type == SYM_ENUM)
+		type = type->ctype.base_type;
+	return type->type;
+}
+
+static inline struct symbol *dmrC_get_nth_symbol(struct symbol_list *list, unsigned int idx)
+{
+	return (struct symbol *)ptrlist_nth_entry((struct ptr_list *)list, idx);
+}
+
+static inline struct symbol *dmrC_lookup_keyword(struct ident *ident,
+					    enum namespace_type ns)
+{
+	if (!ident->keyword)
+		return NULL;
+	return dmrC_lookup_symbol(ident, ns);
+}
+
+static inline void dmrC_concat_symbol_list(struct symbol_list *from, struct symbol_list **to)
+{
+	ptrlist_concat((struct ptr_list *)from, (struct ptr_list **) to);
+}
+
+static inline void dmrC_add_symbol(struct dmr_C *C, struct symbol_list **list, struct symbol *sym)
+{
+	ptrlist_add((struct ptr_list**)list, sym, &C->ptrlist_allocator);
+}
+
+static inline int dmrC_symbol_list_size(struct symbol_list *list)
+{
+	return ptrlist_size((struct ptr_list *)list);
+}
+
+static inline void dmrC_concat_context_list(struct context_list *from,
+					    struct context_list **to)
+{
+	ptrlist_concat((struct ptr_list *)from, (struct ptr_list **)to);
+}
+
+static inline void dmrC_add_context(struct dmr_C *C, struct context_list **list,
+				    struct context *ctx)
+{
+	ptrlist_add((struct ptr_list **)list, ctx, &C->ptrlist_allocator);
+}
+
+static inline int dmrC_is_prototype(struct symbol *sym)
+{
+	if (sym->type == SYM_NODE)
+		sym = sym->ctype.base_type;
+	return sym && sym->type == SYM_FN && !sym->stmt;
+}
+
+#define dmrC_is_restricted_type(type) (dmrC_get_sym_type(type) == SYM_RESTRICT)
+#define dmrC_is_fouled_type(type) (dmrC_get_sym_type(type) == SYM_FOULED)
+#define dmrC_is_bitfield_type(type) (dmrC_get_sym_type(type) == SYM_BITFIELD)
+
+extern void dmrC_create_fouled(struct global_symbols_t *S, struct symbol *type);
+extern struct symbol *dmrC_befoul(struct global_symbols_t *S, struct symbol *type);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/target.c

@@ -0,0 +1,88 @@
+/*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <port.h>
+#include <symbol.h>
+#include <target.h>
+
+enum dummy { DUMMY };
+
+void dmrC_init_target(struct dmr_C *C) {
+
+  struct target_t *t = (struct target_t *)calloc(1, sizeof(struct target_t));
+
+  /*
+  * For "__attribute__((aligned))"
+  */
+  t->max_alignment = 16;
+
+  /*
+  * Integer data types
+  */
+  t->bits_in_bool = 1;
+  t->bits_in_char = 8;
+  t->bits_in_short = 16;
+  t->bits_in_int = 32;
+  t->bits_in_long = sizeof(long) * t->bits_in_char;
+  t->bits_in_longlong = 64;
+  t->bits_in_longlonglong = 128;
+
+  t->bits_in_wchar = 32;
+  t->max_int_alignment = __alignof__(long long); 
+
+  /*
+  * Floating point data types
+  */
+  t->bits_in_float = 32;
+  t->bits_in_double = 64;
+  t->bits_in_longdouble = sizeof(long double) * t->bits_in_char;
+
+  t->max_fp_alignment = 8;
+
+  /*
+  * Pointer data type
+  */
+  t->bits_in_pointer = sizeof(void *) * t->bits_in_char;
+  t->pointer_alignment = __alignof__(void *);
+
+  /*
+  * Enum data types
+  */
+  t->bits_in_enum = 32;
+  t->enum_alignment = 4;
+
+  C->target = t;
+}
+
+void dmrC_destroy_target(struct dmr_C *C) {
+	free(C->target);
+	C->target = NULL;
+}
+

dmr_c/src/target.h

@@ -0,0 +1,85 @@
+#ifndef DMR_C_TARGET_H
+#define DMR_C_TARGET_H
+/*
+* sparse/target.h
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#include <lib.h>
+
+struct target_t {
+  struct symbol *size_t_ctype;
+  struct symbol *ssize_t_ctype;
+
+  /*
+  * For "__attribute__((aligned))"
+  */
+  int max_alignment;
+
+  /*
+  * Integer data types
+  */
+  int bits_in_bool;
+  int bits_in_char;
+  int bits_in_short;
+  int bits_in_int;
+  int bits_in_long;
+  int bits_in_longlong;
+  int bits_in_longlonglong;
+
+  int bits_in_wchar;
+  int max_int_alignment;
+
+  /*
+  * Floating point data types
+  */
+  int bits_in_float;
+  int bits_in_double;
+  int bits_in_longdouble;
+
+  int max_fp_alignment;
+
+  /*
+  * Pointer data type
+  */
+  int bits_in_pointer;
+  int pointer_alignment;
+
+  /*
+  * Enum data types
+  */
+  int bits_in_enum;
+  int enum_alignment;
+};
+
+/*
+* Helper functions for converting bits to bytes and vice versa.
+*/
+
+static inline int dmrC_bits_to_bytes(const struct target_t *target, int bits) {
+  return bits >= 0 ? (bits + target->bits_in_char - 1) / target->bits_in_char : -1;
+}
+
+static inline int dmrC_bytes_to_bits(const struct target_t *target, int bytes) {
+  return bytes * target->bits_in_char;
+}
+
+static inline unsigned long long dmrC_array_element_offset(const struct target_t *target, unsigned int base_bits, int idx)
+{
+	int fragment = base_bits % target->bits_in_char;
+	if (fragment)
+		base_bits += target->bits_in_char - fragment;
+	return base_bits * idx;
+}
+
+extern void dmrC_init_target(struct dmr_C *C);
+extern void dmrC_destroy_target(struct dmr_C *C);
+
+
+#endif

dmr_c/src/token.h

@@ -0,0 +1,294 @@
+/*
+* Basic tokenization structures. NOTE! Those tokens had better
+* be pretty small, since we're going to keep them all in memory
+* indefinitely.
+*
+* Copyright (C) 2003 Transmeta Corp.
+*               2003 Linus Torvalds
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+*/
+/*
+* This version is part of the dmr_c project.
+* Copyright (C) 2017 Dibyendu Majumdar
+*/
+
+#ifndef DMR_C_TOKENIZER_H
+#define DMR_C_TOKENIZER_H
+
+#include <lib.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * This describes the pure lexical elements (tokens), with
+ * no semantic meaning. In other words, an identifier doesn't
+ * have a type or meaning, it is only a specific string in
+ * the input stream.
+ *
+ * Semantic meaning is handled elsewhere.
+ */
+
+enum constantfile {
+  CONSTANT_FILE_MAYBE,    // To be determined, not inside any #ifs in this file
+  CONSTANT_FILE_IFNDEF,   // To be determined, currently inside #ifndef
+  CONSTANT_FILE_NOPE,     // No
+  CONSTANT_FILE_YES       // Yes
+};
+
+struct stream {
+	int fd;
+	const char *name;
+	const char *path; // input-file path - see set_stream_include_path()
+	const char **next_path;
+
+	/* Use these to check for "already parsed" */
+	enum constantfile constant;
+	int dirty, next_stream, once;
+	struct ident *protect;
+	struct token *ifndef;
+	struct token *top_if;
+};
+
+struct ident {
+	struct ident *next;	/* Hash chain of identifiers */
+	struct symbol *symbols;	/* Pointer to semantic meaning list */
+	unsigned char len;	/* Length of identifier name */
+	unsigned char tainted:1,
+	              reserved:1,
+		      keyword:1;
+	char name[];		/* Actual identifier */
+};
+DECLARE_PTR_LIST(ident_list, struct ident);
+
+enum e_token_type {
+	TOKEN_EOF,
+	TOKEN_ERROR,
+	TOKEN_IDENT,
+	TOKEN_ZERO_IDENT,
+	TOKEN_NUMBER,
+	TOKEN_CHAR,
+	TOKEN_CHAR_EMBEDDED_0,
+	TOKEN_CHAR_EMBEDDED_1,
+	TOKEN_CHAR_EMBEDDED_2,
+	TOKEN_CHAR_EMBEDDED_3,
+	TOKEN_WIDE_CHAR,
+	TOKEN_WIDE_CHAR_EMBEDDED_0,
+	TOKEN_WIDE_CHAR_EMBEDDED_1,
+	TOKEN_WIDE_CHAR_EMBEDDED_2,
+	TOKEN_WIDE_CHAR_EMBEDDED_3,
+	TOKEN_STRING,
+	TOKEN_WIDE_STRING,
+	TOKEN_SPECIAL,
+	TOKEN_STREAMBEGIN,
+	TOKEN_STREAMEND,
+	TOKEN_MACRO_ARGUMENT,
+	TOKEN_STR_ARGUMENT,
+	TOKEN_QUOTED_ARGUMENT,
+	TOKEN_CONCAT,
+	TOKEN_GNU_KLUDGE,
+	TOKEN_UNTAINT,
+	TOKEN_ARG_COUNT,
+	TOKEN_IF,
+	TOKEN_SKIP_GROUPS,
+	TOKEN_ELSE,
+};
+
+/* Combination tokens */
+#define COMBINATION_STRINGS {	\
+	"+=", "++",		\
+	"-=", "--", "->",	\
+	"*=",			\
+	"/=",			\
+	"%=",			\
+	"<=", ">=",		\
+	"==", "!=",		\
+	"&&", "&=",		\
+	"||", "|=",		\
+	"^=", "##",		\
+	"<<", ">>", "..",	\
+	"<<=", ">>=", "...",	\
+	"",			\
+	"<", ">", "<=", ">="	\
+}
+
+extern unsigned char dmrC_combinations_[][4];
+
+enum special_token {
+	SPECIAL_BASE = 256,
+	SPECIAL_ADD_ASSIGN = SPECIAL_BASE,
+	SPECIAL_INCREMENT,
+	SPECIAL_SUB_ASSIGN,
+	SPECIAL_DECREMENT,
+	SPECIAL_DEREFERENCE,
+	SPECIAL_MUL_ASSIGN,
+	SPECIAL_DIV_ASSIGN,
+	SPECIAL_MOD_ASSIGN,
+	SPECIAL_LTE,
+	SPECIAL_GTE,
+	SPECIAL_EQUAL,
+	SPECIAL_NOTEQUAL,
+	SPECIAL_LOGICAL_AND,
+	SPECIAL_AND_ASSIGN,
+	SPECIAL_LOGICAL_OR,
+	SPECIAL_OR_ASSIGN,
+	SPECIAL_XOR_ASSIGN,
+	SPECIAL_HASHHASH,
+	SPECIAL_LEFTSHIFT,
+	SPECIAL_RIGHTSHIFT,
+	SPECIAL_DOTDOT,
+	SPECIAL_SHL_ASSIGN,
+	SPECIAL_SHR_ASSIGN,
+	SPECIAL_ELLIPSIS,
+	SPECIAL_ARG_SEPARATOR,
+	SPECIAL_UNSIGNED_LT,
+	SPECIAL_UNSIGNED_GT,
+	SPECIAL_UNSIGNED_LTE,
+	SPECIAL_UNSIGNED_GTE,
+};
+
+struct string {
+	unsigned int length:31;
+	unsigned int immutable:1;
+	char data[];
+};
+
+/* will fit into 32 bits */
+struct argcount {
+	unsigned normal:10;
+	unsigned quoted:10;
+	unsigned str:10;
+	unsigned vararg:1;
+};
+
+/*
+ * This is a very common data structure, it should be kept
+ * as small as humanly possible. Big (rare) types go as
+ * pointers.
+ */
+struct token {
+	struct position pos;
+	struct token *next;
+	union {
+		const char *number;
+		struct ident *ident;
+		unsigned int special;
+		struct string *string;
+		int argnum;
+		struct argcount count;
+		char embedded[4];
+	};
+};
+
+static inline struct token *dmrC_containing_token(struct token **p)
+{
+	void *addr = (char *)p - ((char *)&((struct token *)0)->next - (char *)0);
+	return (struct token *)addr;
+}
+
+struct tokenizer_state_t {
+	unsigned int tabstop;
+	int input_stream_nr;
+	struct stream *input_streams;
+	int input_streams_allocated;
+	char special[256];		   // identifies CR LF TAB
+	long cclass[257];		   // character class
+	unsigned char hash_results[32][2]; // hashes compound operators
+	int code[32]; // token values for compound operators
+	char special_buffer[4];
+	char ident_buffer[256];
+	char string_buffer[4 * MAX_STRING + 3];
+	char char_buffer[MAX_STRING + 4];
+	char quote_buffer[2 * MAX_STRING + 6];
+	char token_buffer[256];
+	char quoted_token_buffer[256];
+	char number_buffer[4095];
+	char string_buffer2[MAX_STRING];
+	struct ident **hash_table;
+	int ident_hit, ident_miss, idents;
+	const char **includepath;
+};
+
+
+#define dmrC_token_type(x) ((x)->pos.type)
+
+/*
+ * Last token in the stream - points to itself.
+ * This allows us to not test for NULL pointers
+ * when following the token->next chain..
+ */
+extern struct token dmrC_eof_token_entry_;
+#define dmrC_eof_token(x) ((x) == &dmrC_eof_token_entry_)
+extern void dmrC_init_tokenizer(struct dmr_C *C);
+extern void dmrC_destroy_tokenizer(struct dmr_C *C);
+
+extern int dmrC_init_stream(struct dmr_C *C, const char *name, int fd,
+	const char **next_path);
+extern const char *dmrC_stream_name(struct dmr_C *C, int stream);
+extern struct ident *dmrC_hash_ident(struct dmr_C *C, struct ident *ident);
+extern struct ident *dmrC_built_in_ident(struct dmr_C *C, const char *name);
+extern struct token *dmrC_built_in_token(struct dmr_C *C, int stream, struct ident *ident);
+extern const char *dmrC_show_special(struct dmr_C *C, int val);
+extern const char *dmrC_show_ident(struct dmr_C *C, const struct ident *ident);
+extern const char *dmrC_show_string(struct dmr_C *C, const struct string *string);
+extern const char *dmrC_show_token(struct dmr_C *C, const struct token *token);
+extern const char *dmrC_quote_token(struct dmr_C *C, const struct token *token);
+extern int *dmrC_hash_stream(const char *name);
+extern struct token *dmrC_tokenize(struct dmr_C *C, const char *name, int fd,
+	struct token *endtoken, const char **next_path);
+/* This function assumes that stream 0 is being used - so it is not suitable
+   for general use */
+extern struct token *dmrC_tokenize_buffer(struct dmr_C *C, unsigned char *buffer,
+	unsigned long size,
+	struct token **endtoken);
+/* This version allows a named stream to be created */
+extern struct token *dmrC_tokenize_buffer_stream(struct dmr_C *C,
+						 const char *name,
+						 unsigned char *buffer,
+						 unsigned long size,
+						 struct token **endtoken);
+extern void dmrC_show_identifier_stats(struct dmr_C *C);
+extern struct token *dmrC_preprocess(struct dmr_C *C, struct token *);
+extern void dmrC_init_preprocessor_state(struct dmr_C *C);
+static inline int dmrC_match_op(struct token *token, unsigned int op)
+{
+	return token->pos.type == TOKEN_SPECIAL && token->special == op;
+}
+
+static inline int dmrC_match_ident(struct token *token, struct ident *id)
+{
+	return token->pos.type == TOKEN_IDENT && token->ident == id;
+}
+
+static inline void dmrC_add_ident(struct dmr_C *C, struct ident_list **list, struct ident *ident)
+{
+	ptrlist_add((struct ptr_list **)list, ident, &C->ptrlist_allocator);
+}
+
+extern int dmrC_test_tokenizer();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

dmr_c/src/unssa.c

@@ -0,0 +1,148 @@
+/*
+ * UnSSA - translate the SSA back to normal form.
+ *
+ * For now it's done by replacing to set of copies:
+ * 1) For each phi-node, replace all their phisrc by copies to a common
+ *    temporary.
+ * 2) Replace all the phi-nodes by copies of the temporaries to the phi-node target.
+ *    This is node to preserve the semantic of the phi-node (they should all "execute"
+ *    simultaneously on entry in the basic block in which they belong).
+ *
+ * This is similar to the "Sreedhar method I" except that the copies to the
+ * temporaries are not placed at the end of the predecessor basic blocks, but
+ * at the place where the phi-node operands are defined.
+ * This is particulary easy since these copies are essentialy already present
+ * as the corresponding OP_PHISOURCE.
+ *
+ * While very simple this method create a lot more copies that really necessary.
+ * We eliminate some of these copies but most probably most of them are still
+ * useless.
+ * Ideally, "Sreedhar method III" should be used:
+ * "Translating Out of Static Single Assignment Form", V. C. Sreedhar, R. D.-C. Ju,
+ * D. M. Gillies and V. Santhanam.  SAS'99, Vol. 1694 of Lecture Notes in Computer
+ * Science, Springer-Verlag, pp. 194-210, 1999.
+ * But for this we need precise liveness, on each %phi and not only on OP_PHI's
+ * target pseudos.
+ *
+ * Copyright (C) 2005 Luc Van Oostenryck
+ */
+
+#include <assert.h>
+
+#include <port.h>
+#include <lib.h>
+#include <linearize.h>
+#include <allocate.h>
+#include <flow.h>
+
+static inline int nbr_pseudo_users(pseudo_t p)
+{
+	return ptrlist_size((struct ptr_list *)p->users);
+}
+
+static int simplify_phi_node(struct dmr_C *C, struct instruction *phi, pseudo_t tmp)
+{
+	pseudo_t target = phi->target;
+	struct pseudo_user *pu;
+	pseudo_t src;
+
+	// verify if this phi can be simplified
+	FOR_EACH_PTR(phi->phi_list, src) {
+		struct instruction *def = src->def;
+
+		if (!def)
+			continue;
+		if (def->bb == phi->bb)
+			return 0;
+	} END_FOR_EACH_PTR(src);
+
+	// no need to make a copy of this one
+	// -> replace the target pseudo by the tmp
+	FOR_EACH_PTR(target->users, pu) {
+		dmrC_use_pseudo(C, pu->insn, tmp, pu->userp);
+	} END_FOR_EACH_PTR(pu);
+
+	phi->bb = NULL;
+	return 1;
+}
+
+static void replace_phi_node(struct dmr_C *C, struct instruction *phi)
+{
+	pseudo_t tmp;
+	pseudo_t p;
+
+	tmp = dmrC_alloc_pseudo(C, NULL);
+	tmp->type = phi->target->type;
+	tmp->ident = phi->target->ident;
+	tmp->def = NULL;		// defined by all the phisrc
+
+	// can we avoid to make of copy?
+	simplify_phi_node(C, phi, tmp);
+
+	// rewrite all it's phi_src to copy to a new tmp
+	FOR_EACH_PTR(phi->phi_list, p) {
+		struct instruction *def = p->def;
+		pseudo_t src;
+
+		if (p == VOID_PSEUDO(C))
+			continue;
+
+		assert(def->opcode == OP_PHISOURCE);
+
+		def->opcode = OP_COPY;
+		def->target = tmp;
+
+		// can we eliminate the copy?
+		src = def->phi_src;
+		if (src->type != PSEUDO_REG)
+			continue;
+		switch (nbr_pseudo_users(src)) {
+			struct instruction *insn;
+		case 1:
+			insn = src->def;
+			if (!insn)
+				break;
+			insn->target = tmp;
+		case 0:
+			dmrC_kill_instruction(C, def);
+			def->bb = NULL;
+		}
+	} END_FOR_EACH_PTR(p);
+
+	if (!phi->bb)
+		return;
+
+	// rewrite the phi node:
+	//	phi	%rt, ...
+	// to:
+	//	copy	%rt, %tmp
+	phi->opcode = OP_COPY;
+	dmrC_use_pseudo(C, phi, tmp, &phi->src);
+}
+
+static void rewrite_phi_bb(struct dmr_C *C, struct basic_block *bb)
+{
+	struct instruction *insn;
+
+	// Replace all the phi-nodes by copies of a temporary
+	// (which represent the set of all the %phi that feed them).
+	// The target pseudo doesn't change.
+	FOR_EACH_PTR(bb->insns, insn) {
+		if (!insn->bb)
+			continue;
+		if (insn->opcode != OP_PHI)
+			continue;
+		replace_phi_node(C, insn);
+	} END_FOR_EACH_PTR(insn);
+}
+
+int dmrC_unssa(struct dmr_C *C, struct entrypoint *ep)
+{
+	struct basic_block *bb;
+
+	FOR_EACH_PTR(ep->bbs, bb) {
+		rewrite_phi_bb(C, bb);
+	} END_FOR_EACH_PTR(bb);
+
+	return 0;
+}

dmr_c/src/walksymbol.c

@@ -0,0 +1,868 @@
+#include <ctype.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <walksymbol.h>
+
+static void walk_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor);
+static void walk_symbol_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_assignment_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_binary_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_preop_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_postop_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_call_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_cast_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_conditional_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_label_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor);
+static void walk_initializer_expression(struct dmr_C *C, struct expression *expr, struct symbol *ctype,
+					struct symbol_visitor *visitor);
+static void walk_label(struct dmr_C *C, struct symbol *label, struct symbol_visitor *visitor);
+static void walk_return_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor);
+static void walk_iterator_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor);
+static void walk_switch_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor);
+
+void walk_return_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor)
+{
+	struct expression *expr = stmt->ret_value;
+	struct symbol *target = stmt->ret_target;
+
+	if (expr && expr->ctype) {
+		walk_expression(C, expr, visitor);
+	}
+	visitor->begin_statement(visitor->data, STMT_GOTO);
+	dmrC_walk_symbol(C, target, visitor);
+	visitor->end_statement(visitor->data);
+}
+
+void walk_label(struct dmr_C *C, struct symbol *label, struct symbol_visitor *visitor)
+{
+	visitor->begin_label(visitor->data, dmrC_show_ident(C, label->ident));
+	dmrC_walk_symbol(C, label, visitor);
+	visitor->end_label(visitor->data);
+}
+
+void walk_iterator_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor)
+{
+	struct statement *pre_statement = stmt->iterator_pre_statement;
+	struct expression *pre_condition = stmt->iterator_pre_condition;
+	struct statement *statement = stmt->iterator_statement;
+	struct statement *post_statement = stmt->iterator_post_statement;
+	struct expression *post_condition = stmt->iterator_post_condition;
+	dmrC_walk_symbol_list(C, stmt->iterator_syms, visitor);
+	if (pre_statement) {
+		visitor->begin_iterator_prestatement(visitor->data);
+		walk_statement(C, pre_statement, visitor);
+		visitor->end_iterator_prestatement(visitor->data);
+	}
+	if (pre_condition) {
+		visitor->begin_iterator_precondition(visitor->data);
+		walk_expression(C, pre_condition, visitor);
+		visitor->end_iterator_precondition(visitor->data);
+	}
+	visitor->begin_iterator_statement(visitor->data);
+	walk_statement(C, statement, visitor);
+	visitor->end_iterator_statement(visitor->data);
+	if (stmt->iterator_continue->used) {
+		walk_label(C, stmt->iterator_continue, visitor);
+	}
+	if (post_statement) {
+		visitor->begin_iterator_poststatement(visitor->data);
+		walk_statement(C, post_statement, visitor);
+		visitor->end_iterator_poststatement(visitor->data);
+	}
+	if (post_condition) {
+		visitor->begin_iterator_postcondition(visitor->data);
+		walk_expression(C, post_condition, visitor);
+		visitor->end_iterator_postcondition(visitor->data);
+	}
+	if (stmt->iterator_break->used) {
+		walk_label(C, stmt->iterator_break, visitor);
+	}
+}
+
+void walk_switch_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor)
+{
+	walk_expression(C, stmt->switch_expression, visitor);
+	struct symbol *sym;
+
+	/*
+	 * Debugging only: Check that the case list is correct
+	 * by printing it out.
+	 *
+	 * This is where a _real_ back-end would go through the
+	 * cases to decide whether to use a lookup table or a
+	 * series of comparisons etc
+	 */
+	FOR_EACH_PTR(stmt->switch_case->symbol_list, sym)
+	{
+		struct statement *case_stmt = sym->stmt;
+		struct expression *expr = case_stmt->case_expression;
+		struct expression *to = case_stmt->case_to;
+
+		if (!expr) {
+			visitor->begin_default_case(visitor->data);
+		} else {
+			if (expr->type == EXPR_VALUE) {
+				if (to) {
+					if (to->type == EXPR_VALUE) {
+						visitor->begin_case_range(visitor->data, expr->value, to->value);
+					}
+				} else {
+					visitor->begin_case_value(visitor->data, expr->value);
+				}
+			}
+		}
+		dmrC_walk_symbol(C, sym, visitor);
+		visitor->end_case(visitor->data);
+	}
+	END_FOR_EACH_PTR(sym);
+
+	walk_statement(C, stmt->switch_statement, visitor);
+
+	if (stmt->switch_break->used)
+		walk_label(C, stmt->switch_break, visitor);
+}
+
+void walk_statement(struct dmr_C *C, struct statement *stmt, struct symbol_visitor *visitor)
+{
+	if (!stmt)
+		return;
+
+	visitor->begin_statement(visitor->data, stmt->type);
+	switch (stmt->type) {
+	case STMT_DECLARATION:
+		dmrC_walk_symbol_list(C, stmt->declaration, visitor);
+		break;
+	case STMT_RETURN:
+		walk_return_statement(C, stmt, visitor);
+		break;
+	case STMT_COMPOUND: {
+		struct statement *s;
+		// if (stmt->inline_fn) {
+		//	dmrC_show_statement(C, stmt->args);
+		//	printf("\tbegin_inline \t%s\n", dmrC_show_ident(C,
+		// stmt->inline_fn->ident));
+		//}
+		FOR_EACH_PTR(stmt->stmts, s) { walk_statement(C, s, visitor); }
+		END_FOR_EACH_PTR(s);
+		if (stmt->ret) {
+			walk_label(C, stmt->ret, visitor);
+		}
+		// if (stmt->inline_fn)
+		//	printf("\tend_inlined\t%s\n", dmrC_show_ident(C,
+		// stmt->inline_fn->ident));
+		break;
+	}
+
+	case STMT_EXPRESSION:
+		walk_expression(C, stmt->expression, visitor);
+		break;
+	case STMT_IF: {
+		struct expression *cond = stmt->if_conditional;
+		walk_expression(C, cond, visitor);
+		visitor->begin_if_then(visitor->data);
+		walk_statement(C, stmt->if_true, visitor);
+		visitor->end_if_then(visitor->data);
+		if (stmt->if_false) {
+			visitor->begin_if_else(visitor->data);
+			walk_statement(C, stmt->if_false, visitor);
+			visitor->end_if_else(visitor->data);
+		}
+		break;
+	}
+	case STMT_SWITCH:
+		walk_switch_statement(C, stmt, visitor);
+		break;
+
+	case STMT_CASE:
+		walk_label(C, stmt->case_label, visitor);
+		walk_statement(C, stmt->case_statement, visitor);
+		break;
+
+	case STMT_ITERATOR:
+		walk_iterator_statement(C, stmt, visitor);
+		break;
+
+	case STMT_NONE:
+		break;
+
+	case STMT_LABEL:
+		visitor->begin_label(visitor->data, dmrC_show_ident(C, stmt->label_identifier->ident));
+		dmrC_walk_symbol(C, stmt->label_identifier, visitor);
+		visitor->end_label(visitor->data);
+		walk_statement(C, stmt->label_statement, visitor);
+		break;
+
+	case STMT_GOTO:
+		if (stmt->goto_expression) {
+			walk_expression(C, stmt->goto_expression, visitor);
+		} else {
+			dmrC_walk_symbol(C, stmt->goto_label, visitor);
+		}
+		break;
+	case STMT_ASM:
+		// printf("\tasm( .... )\n");
+		break;
+	case STMT_CONTEXT: {
+		// int val = dmrC_show_expression(C, stmt->expression);
+		// printf("\tcontext( %d )\n", val);
+		break;
+	}
+	case STMT_RANGE: {
+		// int val = dmrC_show_expression(C, stmt->range_expression);
+		// int low = dmrC_show_expression(C, stmt->range_low);
+		// int high = dmrC_show_expression(C, stmt->range_high);
+		// printf("\trange( %d %d-%d)\n", val, low, high);
+		break;
+	}
+	}
+	visitor->end_statement(visitor->data);
+}
+
+void walk_symbol_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	assert(expr->type == EXPR_SYMBOL);
+	struct symbol *sym = expr->symbol;
+	if (!sym)
+		return;
+	dmrC_walk_symbol(C, sym, visitor);
+}
+
+void walk_assignment_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	assert(expr->type == EXPR_ASSIGNMENT);
+	if (!expr->ctype)
+		return;
+
+	int op = expr->op;
+	visitor->begin_assignment_expression(visitor->data, expr->type, op);
+	walk_expression(C, expr->left, visitor);
+	walk_expression(C, expr->right, visitor);
+	visitor->end_assignment_expression(visitor->data);
+}
+
+void walk_binary_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	assert(expr->type == EXPR_BINOP || expr->type == EXPR_COMPARE || expr->type == EXPR_LOGICAL);
+	if (!expr->ctype)
+		return;
+
+	int op = expr->op;
+	visitor->begin_binop_expression(visitor->data, expr->type, op);
+	walk_expression(C, expr->left, visitor);
+	walk_expression(C, expr->right, visitor);
+	visitor->end_binop_expression(visitor->data);
+}
+
+void walk_preop_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	assert(expr->type == EXPR_PREOP);
+	visitor->begin_preop_expression(visitor->data, expr->type, expr->op);
+	walk_expression(C, expr->unop, visitor);
+	visitor->end_preop_expression(visitor->data);
+}
+
+void walk_postop_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	assert(expr->type == EXPR_POSTOP);
+	visitor->begin_postop_expression(visitor->data, expr->type, expr->op);
+	walk_expression(C, expr->unop, visitor);
+	visitor->end_postop_expression(visitor->data);
+}
+
+void walk_call_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	assert(expr->type == EXPR_CALL);
+	if (!expr->ctype) {
+		return;
+	}
+	struct symbol *direct;
+	struct expression *arg, *fn;
+	fn = expr->fn;
+
+	/* Remove dereference, if any */
+	direct = NULL;
+	if (fn->type == EXPR_PREOP) {
+		if (fn->unop->type == EXPR_SYMBOL) {
+			struct symbol *sym = fn->unop->symbol;
+			if (sym->ctype.base_type->type == SYM_FN)
+				direct = sym;
+		}
+	}
+	if (direct && !direct->aux) {
+		dmrC_walk_symbol(C, direct, visitor);
+	}
+	if (direct) {
+		visitor->begin_direct_call_expression(visitor->data, expr->type, dmrC_show_ident(C, direct->ident));
+	} else {
+		visitor->begin_indirect_call_expression(visitor->data, expr->type);
+		walk_expression(C, fn, visitor);
+	}
+	int n = 0;
+	FOR_EACH_PTR(expr->args, arg)
+	{
+		visitor->begin_callarg_expression(visitor->data, expr->type, ++n);
+		walk_expression(C, arg, visitor);
+		visitor->end_callarg_expression(visitor->data);
+	}
+	END_FOR_EACH_PTR(arg);
+	visitor->end_call_expression(visitor->data);
+}
+
+void walk_cast_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	struct symbol *old_type, *new_type;
+	int oldbits, newbits;
+	int is_signed;
+
+	old_type = expr->cast_expression->ctype;
+	new_type = expr->cast_type;
+
+	oldbits = old_type->bit_size;
+	newbits = new_type->bit_size;
+	is_signed = dmrC_is_signed_type(old_type);
+	visitor->begin_cast_expression(visitor->data, expr->type, oldbits, newbits, !is_signed);
+	walk_expression(C, expr->cast_expression, visitor);
+	dmrC_walk_symbol(C, new_type, visitor);
+	visitor->end_cast_expression(visitor->data);
+}
+
+void walk_conditional_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	visitor->begin_conditional_expression(visitor->data, expr->type);
+	walk_expression(C, expr->conditional, visitor);
+	walk_expression(C, expr->cond_true, visitor);
+	walk_expression(C, expr->cond_false, visitor);
+	visitor->end_conditional_expression(visitor->data);
+}
+
+void walk_label_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	visitor->begin_label_expression(visitor->data, expr->type);
+	dmrC_walk_symbol(C, expr->label_symbol, visitor);
+	visitor->end_label_expression(visitor->data);
+}
+
+static void walk_initialization(struct dmr_C *C, struct symbol *sym, struct expression *expr,
+				struct symbol_visitor *visitor)
+{
+	if (!expr->ctype)
+		return;
+	visitor->begin_initialization(visitor->data, expr->type);
+	walk_expression(C, expr, visitor);
+	dmrC_walk_symbol(C, sym, visitor);
+	visitor->end_initialization(visitor->data);
+}
+
+static void walk_position_expression(struct dmr_C *C, struct expression *expr, struct symbol *base,
+				     struct symbol_visitor *visitor)
+{
+	struct symbol *ctype = expr->init_expr->ctype;
+	int bit_offset;
+
+	bit_offset = ctype ? ctype->bit_offset : -1;
+	visitor->begin_expression_position(visitor->data, EXPR_POS, expr->init_offset, bit_offset,
+					   dmrC_show_ident(C, base->ident));
+	walk_expression(C, expr->init_expr, visitor);
+	visitor->end_expression_position(visitor->data);
+}
+
+void walk_initializer_expression(struct dmr_C *C, struct expression *expr, struct symbol *ctype,
+				 struct symbol_visitor *visitor)
+{
+	struct expression *entry;
+
+	FOR_EACH_PTR(expr->expr_list, entry)
+	{
+
+	again:
+		// Nested initializers have their positions already
+		// recursively calculated - just output them too
+		if (entry->type == EXPR_INITIALIZER) {
+			walk_initializer_expression(C, entry, ctype, visitor);
+			continue;
+		}
+
+		// Initializer indexes and identifiers should
+		// have been evaluated to EXPR_POS
+		if (entry->type == EXPR_IDENTIFIER) {
+			visitor->do_expression_identifier(visitor->data, entry->type,
+							  dmrC_show_ident(C, entry->expr_ident));
+			entry = entry->ident_expression;
+			goto again;
+		}
+
+		if (entry->type == EXPR_INDEX) {
+			visitor->do_expression_index(visitor->data, entry->type, entry->idx_from, entry->idx_to);
+			entry = entry->idx_expression;
+			goto again;
+		}
+		if (entry->type == EXPR_POS) {
+			walk_position_expression(C, entry, ctype, visitor);
+			continue;
+		}
+		walk_initialization(C, ctype, entry, visitor);
+	}
+	END_FOR_EACH_PTR(entry);
+}
+
+void walk_expression(struct dmr_C *C, struct expression *expr, struct symbol_visitor *visitor)
+{
+	if (!expr)
+		return;
+	if (!expr->ctype)
+		return;
+
+	visitor->begin_expression(visitor->data, expr->type);
+	switch (expr->type) {
+	case EXPR_CALL:
+		walk_call_expression(C, expr, visitor);
+		break;
+
+	case EXPR_ASSIGNMENT:
+		walk_assignment_expression(C, expr, visitor);
+		break;
+
+	case EXPR_COMMA:
+		// return show_comma(C, expr);
+		break;
+	case EXPR_BINOP:
+	case EXPR_COMPARE:
+	case EXPR_LOGICAL:
+		walk_binary_expression(C, expr, visitor);
+		break;
+	case EXPR_PREOP:
+		walk_preop_expression(C, expr, visitor);
+		break;
+	case EXPR_POSTOP:
+		walk_postop_expression(C, expr, visitor);
+		break;
+	case EXPR_SYMBOL:
+		walk_symbol_expression(C, expr, visitor);
+		break;
+	case EXPR_DEREF:
+	case EXPR_SIZEOF:
+	case EXPR_PTRSIZEOF:
+	case EXPR_ALIGNOF:
+	case EXPR_OFFSETOF:
+		break;
+	case EXPR_CAST:
+	case EXPR_FORCE_CAST:
+	case EXPR_IMPLIED_CAST:
+		walk_cast_expression(C, expr, visitor);
+		break;
+	case EXPR_VALUE:
+		visitor->int_literal(visitor->data, expr->value, expr->ctype->bit_size,
+				     expr->ctype->ctype.modifiers & MOD_UNSIGNED);
+		break;
+	case EXPR_FVALUE:
+		visitor->float_literal(visitor->data, expr->fvalue, expr->ctype->bit_size);
+		break;
+	case EXPR_STRING:
+		visitor->string_literal(visitor->data, dmrC_show_string(C, expr->string));
+		break;
+	case EXPR_INITIALIZER:
+		walk_initializer_expression(C, expr, expr->ctype, visitor);
+		break;
+	case EXPR_SELECT:
+	case EXPR_CONDITIONAL:
+		walk_conditional_expression(C, expr, visitor);
+		break;
+	case EXPR_STATEMENT:
+		// return show_statement_expr(C, expr);
+		break;
+	case EXPR_LABEL:
+		walk_label_expression(C, expr, visitor);
+		break;
+	case EXPR_SLICE:
+		// return show_slice(C, expr);
+		break;
+
+		// None of these should exist as direct expressions: they are
+		// only valid as sub-expressions of initializers.
+	case EXPR_POS:
+	case EXPR_IDENTIFIER:
+	case EXPR_INDEX:
+	case EXPR_TYPE:
+		break;
+	}
+	visitor->end_expression(visitor->data);
+}
+
+void dmrC_walk_symbol_list(struct dmr_C *C, struct symbol_list *list, struct symbol_visitor *visitor)
+{
+	struct symbol *sym;
+
+	FOR_EACH_PTR(list, sym) { dmrC_walk_symbol(C, sym, visitor); }
+	END_FOR_EACH_PTR(sym);
+}
+
+void dmrC_walk_symbol(struct dmr_C *C, struct symbol *sym, struct symbol_visitor *visitor)
+{
+	if (!sym)
+		return;
+	if (sym->type != SYM_BASETYPE) {
+		if (sym->aux) {
+			/* already visited */
+			const char *name = sym->ident ? dmrC_show_ident(C, sym->ident) : "";
+			visitor->reference_symbol(visitor->data, (uint64_t)sym->aux, name);
+			return;
+		}
+		visitor->id++;
+		sym->aux = (void *)visitor->id;
+	}
+	char name[80] = {0};
+	if (sym->ident)
+		snprintf(name, sizeof name, "%s", dmrC_show_ident(C, sym->ident));
+	else if (sym->type == SYM_BASETYPE)
+		snprintf(name, sizeof name, "%s", dmrC_builtin_typename(C, sym));
+	struct symbol_info syminfo = {.id = (uint64_t)sym->aux,
+				      .name = name,
+				      .symbol_namespace = sym->ns,
+				      .symbol_type = sym->type,
+				      .alignment = sym->ctype.alignment,
+				      .pos = sym->pos,
+				      .bit_size = sym->bit_size,
+				      .offset = sym->offset};
+	if (sym->ns == NS_STRUCT && dmrC_is_bitfield_type(sym)) {
+		syminfo.bit_offset = sym->bit_offset;
+	}
+	if (sym->array_size) {
+		syminfo.array_size = dmrC_get_expression_value(C, sym->array_size);
+	}
+
+	visitor->begin_symbol(visitor->data, &syminfo);
+
+	if (sym->type == SYM_STRUCT || sym->type == SYM_UNION) {
+		struct symbol *member;
+		visitor->begin_struct_members(visitor->data, &syminfo);
+		FOR_EACH_PTR(sym->symbol_list, member) { dmrC_walk_symbol(C, member, visitor); }
+		END_FOR_EACH_PTR(member);
+		visitor->end_struct_members(visitor->data);
+	}
+
+	if (sym->type == SYM_FN) {
+		struct symbol *arg;
+		visitor->begin_func_arguments(visitor->data, &syminfo);
+		FOR_EACH_PTR(sym->arguments, arg) { dmrC_walk_symbol(C, arg, visitor); }
+		END_FOR_EACH_PTR(member);
+		visitor->end_func_arguments(visitor->data);
+	}
+
+	// Is there a base type?
+	if (sym->type != SYM_BASETYPE && sym->ctype.base_type) {
+		if (sym->type == SYM_FN)
+			visitor->begin_func_returntype(visitor->data, &syminfo);
+		else
+			visitor->begin_basetype(visitor->data, &syminfo);
+		dmrC_walk_symbol(C, sym->ctype.base_type, visitor);
+		if (sym->type == SYM_FN)
+			visitor->end_func_returntype(visitor->data);
+		else
+			visitor->end_basetype(visitor->data);
+	}
+
+	if (sym->type == SYM_FN) {
+		if (sym->stmt) {
+			visitor->begin_func_body(visitor->data, &syminfo);
+			walk_statement(C, sym->stmt, visitor);
+			visitor->end_func_body(visitor->data);
+		}
+	}
+	if (sym->initializer) {
+		visitor->begin_initializer(visitor->data, &syminfo);
+		walk_expression(C, sym->initializer, visitor);
+		visitor->end_initializer(visitor->data);
+	}
+	visitor->end_symbol(visitor->data);
+}
+
+static void begin_symbol_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_symbol_default(void *data) { (void)data; }
+static void begin_members_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_members_default(void *data) { (void)data; }
+static void begin_arguments_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_arguments_default(void *data) { (void)data; }
+static void reference_symbol_default(void *data, uint64_t id, const char *name)
+{
+	(void)data;
+	(void)id;
+	(void)name;
+}
+static void begin_body_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_body_default(void *data) { (void)data; }
+static void begin_func_returntype_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_func_returntype_default(void *data) { (void)data; }
+static void begin_basetype_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_basetype_default(void *data) { (void)data; }
+static void begin_initializer_default(void *data, struct symbol_info *syminfo)
+{
+	(void)data;
+	(void)syminfo;
+}
+static void end_initializer_default(void *data) { (void)data; }
+static void string_expression_default(void *data, const char *str)
+{
+	(void)data;
+	(void)str;
+}
+static void int_literal_default(void *data, long long value, int bit_size, bool is_unsigned)
+{
+	(void)data;
+	(void)value;
+	(void)bit_size;
+	(void)is_unsigned;
+}
+static void float_literal_default(void *data, long double fvalue, int bit_size)
+{
+	(void)data;
+	(void)fvalue;
+	(void)bit_size;
+}
+static void begin_statement_default(void *data, enum statement_type statement_type)
+{
+	(void)data;
+	(void)statement_type;
+}
+static void end_statement_default(void *data) { (void)data; }
+static void begin_expression_default(void *data, enum expression_type expr_type)
+{
+	(void)data;
+	(void)expr_type;
+}
+static void end_expression_default(void *data) { (void)data; }
+static void begin_assignment_expression_default(void *data, enum expression_type expr_type, int op)
+{
+	(void)data;
+	(void)expr_type;
+	(void)op;
+}
+static void end_assignment_expression_default(void *data) { (void)data; }
+static void begin_binop_expression_default(void *data, enum expression_type expr_type, int op)
+{
+	(void)data;
+	(void)expr_type;
+	(void)op;
+}
+static void end_binop_expression_default(void *data) { (void)data; }
+static void begin_preop_expression_default(void *data, enum expression_type expr_type, int op)
+{
+	(void)data;
+	(void)expr_type;
+	(void)op;
+}
+static void end_preop_expression_default(void *data) { (void)data; }
+static void begin_postop_expression_default(void *data, enum expression_type expr_type, int op)
+{
+	(void)data;
+	(void)expr_type;
+	(void)op;
+}
+static void end_postop_expression_default(void *data) { (void)data; }
+static void begin_direct_call_expression_default(void *data, enum expression_type expr_type, const char *name)
+{
+	(void)data;
+	(void)expr_type;
+	(void)name;
+}
+static void begin_indirect_call_expression_default(void *data, enum expression_type expr_type)
+{
+	(void)data;
+	(void)expr_type;
+}
+static void end_call_expression_default(void *data) { (void)data; }
+
+static void begin_callarg_expression_default(void *data, enum expression_type expr_type, int argpos)
+{
+	(void)data;
+	(void)expr_type;
+	(void)argpos;
+}
+static void end_callarg_expression_default(void *data) { (void)data; }
+
+static void begin_cast_expression_default(void *data, enum expression_type expr_type, int oldbits, int newbits,
+					  bool is_unsigned)
+{
+	(void)data;
+	(void)expr_type;
+	(void)oldbits;
+	(void)newbits;
+	(void)is_unsigned;
+}
+static void end_cast_expression_default(void *data) { (void)data; }
+static void begin_conditional_expression_default(void *data, enum expression_type expr_type)
+{
+	(void)data;
+	(void)expr_type;
+}
+static void end_conditional_expression_default(void *data) { (void)data; }
+static void begin_label_expression_default(void *data, enum expression_type expr_type)
+{
+	(void)data;
+	(void)expr_type;
+}
+static void end_label_expression_default(void *data) { (void)data; }
+static void do_expression_identifier_default(void *data, enum expression_type expr_type, const char *ident)
+{
+	(void)data;
+	(void)expr_type;
+	(void)ident;
+}
+static void do_expression_index_default(void *data, enum expression_type expr_type, unsigned from, unsigned to)
+{
+	(void)data;
+	(void)expr_type;
+	(void)from;
+	(void)to;
+}
+static void begin_expression_position_default(void *data, enum expression_type expr_type, unsigned init_offset,
+					      int bit_offset, const char *ident)
+{
+	(void)data;
+	(void)expr_type;
+	(void)init_offset;
+	(void)ident;
+	(void)bit_offset;
+}
+static void end_expression_position_default(void *data) { (void)data; }
+static void begin_initialization_default(void *data, enum expression_type expr_type)
+{
+	(void)data;
+	(void)expr_type;
+}
+static void end_initialization_default(void *data) { (void)data; }
+static void begin_label_default(void *data, const char *name)
+{
+	(void)data;
+	(void)name;
+}
+static void end_label_default(void *data) { (void)data; }
+static void begin_iterator_prestatement_default(void *data) { (void)data; }
+static void end_iterator_prestatement_default(void *data) { (void)data; }
+static void begin_iterator_precondition_default(void *data) { (void)data; }
+static void end_iterator_precondition_default(void *data) { (void)data; }
+static void begin_iterator_statement_default(void *data) { (void)data; }
+static void end_iterator_statement_default(void *data) { (void)data; }
+static void begin_iterator_postcondition_default(void *data) { (void)data; }
+static void end_iterator_postcondition_default(void *data) { (void)data; }
+static void begin_iterator_poststatement_default(void *data) { (void)data; }
+static void end_iterator_poststatement_default(void *data) { (void)data; }
+static void begin_case_value_default(void *data, long long value)
+{
+	(void)data;
+	(void)value;
+}
+static void begin_case_range_default(void *data, long long from, long long to)
+{
+	(void)data;
+	(void)from;
+	(void)to;
+}
+static void begin_default_case_default(void *data) { (void)data; }
+static void end_case_default(void *data) { (void)data; }
+
+static void begin_if_then_default(void *data) { (void)data; }
+static void end_if_then_default(void *data) { (void)data; }
+static void begin_if_else_default(void *data) { (void)data; }
+static void end_if_else_default(void *data) { (void)data; }
+
+void dmrC_init_symbol_visitor(struct symbol_visitor *visitor)
+{
+	visitor->data = NULL;
+	visitor->id = 0;
+	visitor->begin_symbol = begin_symbol_default;
+	visitor->end_symbol = end_symbol_default;
+	visitor->begin_struct_members = begin_members_default;
+	visitor->end_struct_members = end_members_default;
+	visitor->begin_func_arguments = begin_arguments_default;
+	visitor->end_func_arguments = end_arguments_default;
+	visitor->reference_symbol = reference_symbol_default;
+	visitor->begin_func_body = begin_body_default;
+	visitor->end_func_body = end_body_default;
+	visitor->begin_func_returntype = begin_func_returntype_default;
+	visitor->end_func_returntype = end_func_returntype_default;
+	visitor->begin_basetype = begin_basetype_default;
+	visitor->end_basetype = end_basetype_default;
+	visitor->begin_initializer = begin_initializer_default;
+	visitor->end_initializer = end_initializer_default;
+	visitor->string_literal = string_expression_default;
+	visitor->float_literal = float_literal_default;
+	visitor->int_literal = int_literal_default;
+	visitor->begin_statement = begin_statement_default;
+	visitor->end_statement = end_statement_default;
+	visitor->begin_expression = begin_expression_default;
+	visitor->end_expression = end_expression_default;
+	visitor->begin_assignment_expression = begin_assignment_expression_default;
+	visitor->end_assignment_expression = end_assignment_expression_default;
+	visitor->begin_binop_expression = begin_binop_expression_default;
+	visitor->end_binop_expression = end_binop_expression_default;
+	visitor->begin_preop_expression = begin_preop_expression_default;
+	visitor->end_preop_expression = end_preop_expression_default;
+	visitor->begin_postop_expression = begin_postop_expression_default;
+	visitor->end_postop_expression = end_postop_expression_default;
+	visitor->begin_direct_call_expression = begin_direct_call_expression_default;
+	visitor->begin_indirect_call_expression = begin_indirect_call_expression_default;
+	visitor->end_call_expression = end_call_expression_default;
+	visitor->begin_callarg_expression = begin_callarg_expression_default;
+	visitor->end_callarg_expression = end_callarg_expression_default;
+	visitor->begin_cast_expression = begin_cast_expression_default;
+	visitor->end_cast_expression = end_cast_expression_default;
+	visitor->begin_conditional_expression = begin_conditional_expression_default;
+	visitor->end_conditional_expression = end_conditional_expression_default;
+	visitor->begin_label_expression = begin_label_expression_default;
+	visitor->end_label_expression = end_label_expression_default;
+	visitor->do_expression_identifier = do_expression_identifier_default;
+	visitor->do_expression_index = do_expression_index_default;
+	visitor->begin_expression_position = begin_expression_position_default;
+	visitor->end_expression_position = end_expression_position_default;
+	visitor->begin_initialization = begin_initialization_default;
+	visitor->end_initialization = end_initialization_default;
+	visitor->begin_label = begin_label_default;
+	visitor->end_label = end_label_default;
+	visitor->begin_iterator_prestatement = begin_iterator_prestatement_default;
+	visitor->end_iterator_prestatement = end_iterator_prestatement_default;
+	visitor->begin_iterator_precondition = begin_iterator_precondition_default;
+	visitor->end_iterator_precondition = end_iterator_precondition_default;
+	visitor->begin_iterator_statement = begin_iterator_statement_default;
+	visitor->end_iterator_statement = end_iterator_statement_default;
+	visitor->begin_iterator_postcondition = begin_iterator_postcondition_default;
+	visitor->end_iterator_postcondition = end_iterator_postcondition_default;
+	visitor->begin_iterator_poststatement = begin_iterator_poststatement_default;
+	visitor->end_iterator_poststatement = end_iterator_poststatement_default;
+	visitor->begin_case_value = begin_case_value_default;
+	visitor->begin_case_range = begin_case_range_default;
+	visitor->begin_default_case = begin_default_case_default;
+	visitor->end_case = end_case_default;
+	visitor->begin_if_then = begin_if_then_default;
+	visitor->end_if_then = end_if_then_default;
+	visitor->begin_if_else = begin_if_else_default;
+	visitor->end_if_else = end_if_else_default;
+}

dmr_c/src/walksymbol.h

@@ -0,0 +1,144 @@
+#ifndef DMR_C_PARSETREE_H
+#define DMR_C_PARSETREE_H
+
+#include <allocate.h>
+#include <lib.h>
+#include <expression.h>
+#include <parse.h>
+#include <port.h>
+#include <symbol.h>
+#include <token.h>
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct symbol_info {
+	uint64_t id;
+	enum namespace_type symbol_namespace;
+	enum type symbol_type;
+	const char *name;
+	int bit_size;
+	unsigned long alignment;
+	unsigned int offset;
+	int bit_offset;
+	long long array_size;
+	struct position pos;
+};
+
+struct symbol_visitor {
+	void *data;
+	uint64_t id;
+	void (*begin_symbol)(void *data, struct symbol_info *syminfo);
+	void (*end_symbol)(void *data);
+	void (*begin_struct_members)(void *data, struct symbol_info *syminfo);
+	void (*end_struct_members)(void *data);
+	void (*begin_func_arguments)(void *data, struct symbol_info *syminfo);
+	void (*end_func_arguments)(void *data);
+	void (*reference_symbol)(void *data, uint64_t id, const char *name);
+	void (*begin_func_body)(void *data, struct symbol_info *syminfo);
+	void (*end_func_body)(void *data);
+	void (*begin_func_returntype)(void *data, struct symbol_info *syminfo);
+	void (*end_func_returntype)(void *data);
+	void (*begin_basetype)(void *data, struct symbol_info *syminfo);
+	void (*end_basetype)(void *data);
+	void (*begin_initializer)(void *data, struct symbol_info *syminfo);
+	void (*end_initializer)(void *data);
+	void (*string_literal)(void *data, const char *str);
+	void (*int_literal)(void *data, long long value, int bit_size,
+			    bool is_unsigned);
+	void (*float_literal)(void *data, long double fvalue, int bit_size);
+
+	void (*begin_statement)(void *data, enum statement_type statement_type);
+	void (*end_statement)(void *data);
+
+	void (*begin_expression)(void *data, enum expression_type expr_type);
+	void (*end_expression)(void *data);
+	void (*begin_assignment_expression)(void *data,
+					    enum expression_type expr_type,
+					    int op);
+	void (*end_assignment_expression)(void *data);
+	void (*begin_binop_expression)(void *data,
+				       enum expression_type expr_type, int op);
+	void (*end_binop_expression)(void *data);
+	void (*begin_preop_expression)(void *data,
+				       enum expression_type expr_type, int op);
+	void (*end_preop_expression)(void *data);
+	void (*begin_postop_expression)(void *data,
+					enum expression_type expr_type, int op);
+	void (*end_postop_expression)(void *data);
+
+	void (*begin_direct_call_expression)(void *data,
+					     enum expression_type expr_type,
+					     const char *name);
+	void (*begin_indirect_call_expression)(void *data,
+					       enum expression_type expr_type);
+	void (*end_call_expression)(void *data);
+	void (*begin_callarg_expression)(void *data,
+					 enum expression_type expr_type,
+					 int argpos);
+	void (*end_callarg_expression)(void *data);
+	void (*begin_cast_expression)(void *data,
+				      enum expression_type expr_type,
+				      int oldbits, int newbits,
+				      bool is_unsigned);
+	void (*end_cast_expression)(void *data);
+	void (*begin_conditional_expression)(void *data,
+					     enum expression_type expr_type);
+	void (*end_conditional_expression)(void *data);
+	void (*begin_label_expression)(void *data,
+				       enum expression_type expr_type);
+	void (*end_label_expression)(void *data);
+	void (*do_expression_identifier)(void *data,
+					 enum expression_type expr_type,
+					 const char *ident);
+	void (*do_expression_index)(void *data, enum expression_type expr_type,
+				    unsigned from, unsigned to);
+	void (*begin_expression_position)(void *data,
+					  enum expression_type expr_type,
+					  unsigned init_offset, int bit_offset,
+					  const char *ident);
+	void (*end_expression_position)(void *data);
+	void (*begin_initialization)(void *data,
+				     enum expression_type expr_type);
+	void (*end_initialization)(void *data);
+
+	void (*begin_label)(void *data, const char *name);
+	void (*end_label)(void *data);
+
+	void(*begin_iterator_prestatement)(void *data);
+	void(*end_iterator_prestatement)(void *data);
+	void(*begin_iterator_precondition)(void *data);
+	void(*end_iterator_precondition)(void *data);
+	void(*begin_iterator_statement)(void *data);
+	void(*end_iterator_statement)(void *data);
+	void(*begin_iterator_postcondition)(void *data);
+	void(*end_iterator_postcondition)(void *data);
+	void(*begin_iterator_poststatement)(void *data);
+	void(*end_iterator_poststatement)(void *data);
+
+	void(*begin_case_value)(void *data, long long value);
+	void(*begin_case_range)(void *data, long long from, long long to);
+	void(*begin_default_case)(void *data);
+	void(*end_case)(void *data);
+
+	void(*begin_if_then)(void *data);
+	void(*end_if_then)(void *data);
+	void(*begin_if_else)(void *data);
+	void(*end_if_else)(void *data);
+};
+
+extern void dmrC_init_symbol_visitor(struct symbol_visitor *visitor);
+extern void dmrC_walk_symbol_list(struct dmr_C *C, struct symbol_list *list,
+				  struct symbol_visitor *visitor);
+extern void dmrC_walk_symbol(struct dmr_C *C, struct symbol *sym,
+			     struct symbol_visitor *visitor);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

docker/linux-ubuntu/Dockerfile

@@ -12,15 +12,20 @@ RUN set -x \
     && cd /Software \
     && tar xvf "cmake-3.14.5-Linux-x86_64.tar.gz" \
     && rm -rf "/Software/cmake-3.14.5-Linux-x86_64.tar.gz" \
+    && wget -O "/Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" "http://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" \
+    && cd /Software \
+    && tar xvf "clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" \
+    && rm -rf "/Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz" \
     && mkdir -p /sources \
     && cd /sources \
     && git clone https://github.com/dibyendumajumdar/ravi.git \
     && cd /sources/ravi \
     && mkdir build \
     && cd build \
-    && /Software/cmake-3.14.5-Linux-x86_64/bin/cmake -DCMAKE_INSTALL_PREFIX=/Software/ravi -DCMAKE_BUILD_TYPE=Release .. \
+    && /Software/cmake-3.14.5-Linux-x86_64/bin/cmake -DSTATIC_BUILD=ON -DCMAKE_INSTALL_PREFIX=/Software/ravi -DLLVM_JIT=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_DIR=/Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04/lib/cmake/llvm .. \
     && make install \
     && rm -rf /Software/cmake-3.14.5-Linux-x86_64 \
+    && rm -rf /Software/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04 \
     && rm -rf /sources \
     && apt-get autoremove \
     && apt-get remove -y --purge git wget build-essential \

include/lauxlib.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lauxlib.h $
+** $Id: lauxlib.h,v 1.131.1.1 2017/04/19 17:20:42 roberto Exp $
 ** Auxiliary functions for building Lua libraries
 ** See Copyright Notice in lua.h
 */
@@ -15,12 +15,6 @@
 #include "lua.h"
 
 
-/* global table */
-#define LUA_GNAME	"_G"
-
-
-typedef struct luaL_Buffer luaL_Buffer;
-
 
 /* extra error code for 'luaL_loadfilex' */
 #define LUA_ERRFILE     (LUA_ERRERR+1)
@@ -54,7 +48,6 @@ LUALIB_API const char *(luaL_tolstring) (lua_State *L, int idx, size_t *len);
    api function ravi_typename() */
 LUALIB_API const char *(raviL_tolstring) (lua_State *L, int idx, size_t *len);
 LUALIB_API int (luaL_argerror) (lua_State *L, int arg, const char *extramsg);
-LUALIB_API int (luaL_typeerror) (lua_State *L, int arg, const char *tname);
 LUALIB_API const char *(luaL_checklstring) (lua_State *L, int arg,
                                                           size_t *l);
 LUALIB_API const char *(luaL_optlstring) (lua_State *L, int arg,
@@ -84,7 +77,6 @@ LUALIB_API int (luaL_checkoption) (lua_State *L, int arg, const char *def,
 LUALIB_API int (luaL_fileresult) (lua_State *L, int stat, const char *fname);
 LUALIB_API int (luaL_execresult) (lua_State *L, int stat);
 
-
 /* predefined references */
 #define LUA_NOREF       (-2)
 #define LUA_REFNIL      (-1)
@@ -105,10 +97,8 @@ LUALIB_API lua_State *(luaL_newstate) (void);
 
 LUALIB_API lua_Integer (luaL_len) (lua_State *L, int idx);
 
-LUALIB_API void luaL_addgsub (luaL_Buffer *b, const char *s,
-                                     const char *p, const char *r);
-LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s,
-                                    const char *p, const char *r);
+LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s, const char *p,
+                                                  const char *r);
 
 LUALIB_API void (luaL_setfuncs) (lua_State *L, const luaL_Reg *l, int nup);
 
@@ -135,10 +125,6 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname,
 
 #define luaL_argcheck(L, cond,arg,extramsg)	\
 		((void)((cond) || luaL_argerror(L, (arg), (extramsg))))
-
-#define luaL_argexpected(L,cond,arg,tname)	\
-		((void)((cond) || luaL_typeerror(L, (arg), (tname))))
-
 #define luaL_checkstring(L,n)	(luaL_checklstring(L, (n), NULL))
 #define luaL_optstring(L,n,d)	(luaL_optlstring(L, (n), (d), NULL))
 
@@ -157,10 +143,6 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname,
 #define luaL_loadbuffer(L,s,sz,n)	luaL_loadbufferx(L,s,sz,n,NULL)
 
 
-/* push the value used to represent failure/error */
-#define luaL_pushfail(L)	lua_pushnil(L)
-
-
 /*
 ** {======================================================
 ** Generic Buffer manipulation
@@ -294,6 +276,10 @@ LUALIB_API void  (luaL_setmetatable)(lua_State *L, const char *tname);
 LUALIB_API void *(luaL_testudata)(lua_State *L, int ud, const char *tname);
 LUALIB_API void *(luaL_checkudata)(lua_State *L, int ud, const char *tname);
 
+LUALIB_API int (raviL_build_ast_from_buffer) (lua_State *L, const char *buff, size_t size,
+                                 const char *name, const char *mode);
+
+LUALIB_API int (raviL_dumpast) (lua_State *L);
 LUALIB_API void *ravi_alloc_f(void *msp, void *ptr, size_t osize, size_t nsize);
 
 #endif

include/lfunc.h

@@ -30,28 +30,43 @@
 /* RAVI change; #define MAXUPVAL	255 */
 
 
-#define upisopen(up)	((up)->v != &(up)->u.value)
-
 /*
-** Special "status" for 'luaF_close'
+** Upvalues for Lua closures
 */
+struct UpVal {
+  TValue *v;  /* points to stack or to its own value */
+#ifdef RAVI_DEFER_STATEMENT
+  unsigned int refcount;  /* reference counter */
+  unsigned int flags; /* Used to mark deferred values */
+#else
+  lu_mem refcount;  /* reference counter */
+#endif
+  union {
+    struct {  /* (when open) */
+      UpVal *next;  /* linked list */
+      int touched;  /* mark to avoid cycles with dead threads */
+    } open;
+    TValue value;  /* the value (when closed) */
+  } u;
+};
 
-/* close upvalues without running their closing methods */
-#define NOCLOSINGMETH	(-1)
+#define upisopen(up)	((up)->v != &(up)->u.value)
 
-/* close upvalues running all closing methods in protected mode */
-#define CLOSEPROTECT	(-2)
 
 LUAI_FUNC Proto *luaF_newproto (lua_State *L);
 LUAI_FUNC CClosure *luaF_newCclosure (lua_State *L, int nelems);
 LUAI_FUNC LClosure *luaF_newLclosure (lua_State *L, int nelems);
 LUAI_FUNC void luaF_initupvals (lua_State *L, LClosure *cl);
 LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level);
+#ifdef RAVI_DEFER_STATEMENT
 LUAI_FUNC int luaF_close (lua_State *L, StkId level, int status);
+#else
+LUAI_FUNC void luaF_close (lua_State *L, StkId level);
+#endif
 LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f);
 /* The additional type argument is a Ravi extension */
 LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number,
-                                         int pc, ravi_type_map* type, TString **usertype);
+                                         int pc, ravitype_t* type);
 
 
 #endif

include/lgc.h

@@ -0,0 +1,147 @@
+/*
+** $Id: lgc.h,v 2.91.1.1 2017/04/19 17:39:34 roberto Exp $
+** Garbage Collector
+** See Copyright Notice in lua.h
+*/
+
+#ifndef lgc_h
+#define lgc_h
+
+
+#include "lobject.h"
+#include "lstate.h"
+
+/*
+** Collectable objects may have one of three colors: white, which
+** means the object is not marked; gray, which means the
+** object is marked, but its references may be not marked; and
+** black, which means that the object and all its references are marked.
+** The main invariant of the garbage collector, while marking objects,
+** is that a black object can never point to a white one. Moreover,
+** any gray object must be in a "gray list" (gray, grayagain, weak,
+** allweak, ephemeron) so that it can be visited again before finishing
+** the collection cycle. These lists have no meaning when the invariant
+** is not being enforced (e.g., sweep phase).
+*/
+
+
+
+/* how much to allocate before next GC step */
+#if !defined(GCSTEPSIZE)
+/* ~100 small strings */
+#define GCSTEPSIZE	(cast_int(100 * sizeof(TString)))
+#endif
+
+
+/*
+** Possible states of the Garbage Collector
+*/
+#define GCSpropagate	0
+#define GCSatomic	1
+#define GCSswpallgc	2
+#define GCSswpfinobj	3
+#define GCSswptobefnz	4
+#define GCSswpend	5
+#define GCScallfin	6
+#define GCSpause	7
+
+
+#define issweepphase(g)  \
+	(GCSswpallgc <= (g)->gcstate && (g)->gcstate <= GCSswpend)
+
+
+/*
+** macro to tell when main invariant (white objects cannot point to black
+** ones) must be kept. During a collection, the sweep
+** phase may break the invariant, as objects turned white may point to
+** still-black objects. The invariant is restored when sweep ends and
+** all objects are white again.
+*/
+
+#define keepinvariant(g)	((g)->gcstate <= GCSatomic)
+
+
+/*
+** some useful bit tricks
+*/
+#define resetbits(x,m)		((x) &= cast(lu_byte, ~(m)))
+#define setbits(x,m)		((x) |= (m))
+#define testbits(x,m)		((x) & (m))
+#define bitmask(b)		(1<<(b))
+#define bit2mask(b1,b2)		(bitmask(b1) | bitmask(b2))
+#define l_setbit(x,b)		setbits(x, bitmask(b))
+#define resetbit(x,b)		resetbits(x, bitmask(b))
+#define testbit(x,b)		testbits(x, bitmask(b))
+
+
+/* Layout for bit use in 'marked' field: */
+#define WHITE0BIT	0  /* object is white (type 0) */
+#define WHITE1BIT	1  /* object is white (type 1) */
+#define BLACKBIT	2  /* object is black */
+#define FINALIZEDBIT	3  /* object has been marked for finalization */
+/* bit 7 is currently used by tests (luaL_checkmemory) */
+
+#define WHITEBITS	bit2mask(WHITE0BIT, WHITE1BIT)
+
+
+#define iswhite(x)      testbits((x)->marked, WHITEBITS)
+#define isblack(x)      testbit((x)->marked, BLACKBIT)
+#define isgray(x)  /* neither white nor black */  \
+	(!testbits((x)->marked, WHITEBITS | bitmask(BLACKBIT)))
+
+#define tofinalize(x)	testbit((x)->marked, FINALIZEDBIT)
+
+#define otherwhite(g)	((g)->currentwhite ^ WHITEBITS)
+#define isdeadm(ow,m)	(!(((m) ^ WHITEBITS) & (ow)))
+#define isdead(g,v)	isdeadm(otherwhite(g), (v)->marked)
+
+#define changewhite(x)	((x)->marked ^= WHITEBITS)
+#define gray2black(x)	l_setbit((x)->marked, BLACKBIT)
+
+#define luaC_white(g)	cast(lu_byte, (g)->currentwhite & WHITEBITS)
+
+
+/*
+** Does one step of collection when debt becomes positive. 'pre'/'pos'
+** allows some adjustments to be done only when needed. macro
+** 'condchangemem' is used only for heavy tests (forcing a full
+** GC cycle on every opportunity)
+*/
+#define luaC_condGC(L,pre,pos) \
+	{ if (G(L)->GCdebt > 0) { pre; luaC_step(L); pos;}; \
+	  condchangemem(L,pre,pos); }
+
+/* more often than not, 'pre'/'pos' are empty */
+#define luaC_checkGC(L)		luaC_condGC(L,(void)0,(void)0)
+
+
+#define luaC_barrier(L,p,v) (  \
+	(iscollectable(v) && isblack(p) && iswhite(gcvalue(v))) ?  \
+	luaC_barrier_(L,obj2gco(p),gcvalue(v)) : cast_void(0))
+
+#define luaC_barrierback(L,p,v) (  \
+	(iscollectable(v) && isblack(p) && iswhite(gcvalue(v))) ? \
+	luaC_barrierback_(L,p) : cast_void(0))
+
+#define luaC_objbarrier(L,p,o) (  \
+	(isblack(p) && iswhite(o)) ? \
+	luaC_barrier_(L,obj2gco(p),obj2gco(o)) : cast_void(0))
+
+#define luaC_upvalbarrier(L,uv) ( \
+	(iscollectable((uv)->v) && !upisopen(uv)) ? \
+         luaC_upvalbarrier_(L,uv) : cast_void(0))
+
+LUAI_FUNC void luaC_fix (lua_State *L, GCObject *o);
+LUAI_FUNC void luaC_freeallobjects (lua_State *L);
+LUAI_FUNC void luaC_step (lua_State *L);
+LUAI_FUNC void luaC_runtilstate (lua_State *L, int statesmask);
+LUAI_FUNC void luaC_fullgc (lua_State *L, int isemergency);
+LUAI_FUNC GCObject *luaC_newobj (lua_State *L, int tt, size_t sz);
+LUAI_FUNC void luaC_barrier_ (lua_State *L, GCObject *o, GCObject *v);
+LUAI_FUNC void luaC_barrierback_ (lua_State *L, Table *o);
+LUAI_FUNC void luaC_upvalbarrier_ (lua_State *L, UpVal *uv);
+LUAI_FUNC void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt);
+LUAI_FUNC void luaC_upvdeccount (lua_State *L, UpVal *uv);
+
+
+#endif

include/llex.h

@@ -27,7 +27,11 @@ enum RESERVED {
   /* terminal symbols denoted by reserved words */
   TK_AND = FIRST_RESERVED, TK_BREAK,
   TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
+#ifdef RAVI_DEFER_STATEMENT
   TK_GOTO, TK_IF, TK_IN, TK_LOCAL, TK_DEFER, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
+#else
+  TK_GOTO, TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
+#endif
   TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
   /* other terminal symbols */
   TK_IDIV, TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE,

include/llimits.h

@@ -33,7 +33,6 @@ typedef long l_mem;
 
 /* chars used as small naturals (so that 'char' is reserved for characters) */
 typedef unsigned char lu_byte;
-typedef signed char ls_byte;
 
 
 /* maximum value for size_t */
@@ -52,23 +51,6 @@ typedef signed char ls_byte;
 #define MAX_INT		INT_MAX  /* maximum value of an int */
 
 
-/*
-** floor of the log2 of the maximum signed value for integral type 't'.
-** (That is, maximum 'n' such that '2^n' fits in the given signed type.)
-*/
-#define log2maxs(t)	(sizeof(t) * 8 - 2)
-
-
-/*
-** test whether an unsigned value is a power of 2 (or zero)
-*/
-#define ispow2(x)	(((x) & ((x) - 1)) == 0)
-
-
-/* number of chars of a literal string without the ending \0 */
-#define LL(x)   (sizeof(x)/sizeof(char) - 1)
-
-
 /*
 ** conversion of pointer to unsigned integer:
 ** this is for hashing only; there is no problem if the integer
@@ -102,7 +84,7 @@ typedef LUAI_UACINT l_uacInt;
 #if defined(lua_assert)
 /** RAVI changes */
 #if !defined(RAVI_OPTION_STRING1)
-#define RAVI_OPTION_STRING1 "assertions "
+#define RAVI_OPTION_STRING1 " assertions"
 #endif
 #if !defined(RAVI_OPTION_STRING2)
 #define RAVI_OPTION_STRING2
@@ -114,7 +96,7 @@ typedef LUAI_UACINT l_uacInt;
 #define lua_assert(c)		((void)0)
 #define check_exp(c,e)		(e)
 #define lua_longassert(c)	((void)0)
-/** RAVI changes */
+/** RVAI changes */
 #define RAVI_OPTION_STRING1
 #define RAVI_OPTION_STRING2
 #endif
@@ -139,15 +121,10 @@ typedef LUAI_UACINT l_uacInt;
 #define cast(t, exp)	((t)(exp))
 
 #define cast_void(i)	cast(void, (i))
-#define cast_voidp(i)	cast(void *, (i))
+#define cast_byte(i)	cast(lu_byte, (i))
 #define cast_num(i)	cast(lua_Number, (i))
 #define cast_int(i)	cast(int, (i))
-#define cast_uint(i)	cast(unsigned int, (i))
-#define cast_byte(i)	cast(lu_byte, (i))
 #define cast_uchar(i)	cast(unsigned char, (i))
-#define cast_char(i)	cast(char, (i))
-#define cast_charp(i)	cast(char *, (i))
-#define cast_sizet(i)	cast(size_t, (i))
 
 
 /* cast a signed lua_Integer to lua_Unsigned */
@@ -165,22 +142,6 @@ typedef LUAI_UACINT l_uacInt;
 #endif
 
 
-/*
-** macros to improve jump prediction (used mainly for error handling)
-*/
-#if !defined(likely)
-
-#if defined(__GNUC__)
-#define likely(x)	(__builtin_expect(((x) != 0), 1))
-#define unlikely(x)	(__builtin_expect(((x) != 0), 0))
-#else
-#define likely(x)	(x)
-#define unlikely(x)	(x)
-#endif
-
-#endif
-
-
 /*
 ** non-return type
 */

include/lmem.h

@@ -49,26 +49,10 @@
 
 #define luaM_newobject(L,tag,s)	luaM_realloc_(L, NULL, tag, (s))
 
-/*
- * Note that this will grow the vector by unspecified amount and set the size parameter to new size!
- *
- * v - pointer to memory
- * nelems - number of elements needed
- * size - curent size of v, will be UPDATED!
- * t - element type
- * limit - max number allowed
- * e - for error message?
- */
 #define luaM_growvector(L,v,nelems,size,t,limit,e) \
           if ((nelems)+1 > (size)) \
             ((v)=cast(t *, luaM_growaux_(L,v,&(size),sizeof(t),limit,e)))
 
-/*
- * v - vector - will be updated!
- * oldn - old size
- * n - new size
- * t - element type
- */
 #define luaM_reallocvector(L, v,oldn,n,t) \
    ((v)=cast(t *, luaM_reallocv(L, v, oldn, n, sizeof(t))))
 

include/lobject.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lobject.h $
+** $Id: lobject.h,v 2.117.1.1 2017/04/19 17:39:34 roberto Exp $
 ** Type definitions for Lua objects
 ** See Copyright Notice in lua.h
 */
@@ -27,8 +27,57 @@
 */
 #define LUA_TOTALTAGS	(LUA_TPROTO + 2)
 
+
+/*
+** tags for Tagged Values have the following use of bits:
+** bits 0-3: actual tag (a LUA_T* value)
+** bits 4-5: variant bits
+** bit 6: whether value is collectable
+*/
+
+
+/*
+** LUA_TFUNCTION variants:
+** 0 - Lua function
+** 1 - light C function
+** 2 - regular C function (closure)
+** 4 - fast light C dunction (Ravi extension)
+*/
+
+/* Variant tags for functions */
+#define LUA_TLCL	(LUA_TFUNCTION | (0 << 4))  /* Lua closure */
+#define LUA_TLCF	(LUA_TFUNCTION | (1 << 4))  /* light C function */
+#define LUA_TCCL	(LUA_TFUNCTION | (2 << 4))  /* C closure */
+#define RAVI_TFCF	(LUA_TFUNCTION | (4 << 4))  /* fast light C function */
+
+
+/* Variant tags for strings */
+#define LUA_TSHRSTR	(LUA_TSTRING | (0 << 4))  /* short strings */
+#define LUA_TLNGSTR	(LUA_TSTRING | (1 << 4))  /* long strings */
+
+
+/* Variant tags for numbers */
+#define LUA_TNUMFLT	(LUA_TNUMBER | (0 << 4))  /* float numbers */
+#define LUA_TNUMINT	(LUA_TNUMBER | (1 << 4))  /* integer numbers */
+/** RAVI table subtypes **/
+#define RAVI_TIARRAY (LUA_TTABLE | (1 << 4))  /* Ravi int array */
+#define RAVI_TFARRAY (LUA_TTABLE | (2 << 4))  /* Ravi float array */
+
+
+/* Bit mark for collectable types */
+#define BIT_ISCOLLECTABLE	(1 << 15)
+
+/* mark a tag as collectable */
+#define ctb(t)			((t) | BIT_ISCOLLECTABLE)
+
+
+/*
+** Common type for all collectable objects
+*/
+typedef struct GCObject GCObject;
+
 /* 
-** In Ravi, value type is extended to 16-bits so that we can hold more info.
+** Value type extended to 16-bits so that we can hold more info.
 ** The actual type code is still 1 byte (least significant byte)
 ** and in particular all GC-able type codes must fit into 1 byte because
 ** the GC CommonHeader only allows 1 byte for the type code.
@@ -38,83 +87,23 @@
 typedef uint16_t LuaType;
 
 /*
-** tags for Tagged Values have the following use of bits:
-** bits 0-3: actual tag (a LUA_T* value)
-** bits 4-5: variant bits
-** bit 15: whether value is collectable
+** Common Header for all collectable objects (in macro form, to be
+** included in other objects)
+** Note that tt field is a byte - this means that a GC object's
+** type must have all information in the first byte
 */
+#define CommonHeader	GCObject *next; lu_byte tt; lu_byte marked
 
-/* add variant bits to a type */
-#define makevariant(t,v)	((t) | ((v) << 4))
 
+/*
+** Common type has only the common header
+*/
+struct GCObject {
+  CommonHeader;
+};
 
-/* RAVI: Following are the types we will use
-** use in parsing. The rationale for types is
-** performance - as of now these are the only types that
-** we care about from a performance point of view - if any
-** other types appear then they are all treated as ANY
-**/
 
-typedef enum {
-  RAVI_TI_NIL,
-  RAVI_TI_FALSE,
-  RAVI_TI_TRUE,
-  RAVI_TI_INTEGER,
-  RAVI_TI_FLOAT,
-  RAVI_TI_INTEGER_ARRAY,
-  RAVI_TI_FLOAT_ARRAY,
-  RAVI_TI_TABLE,
-  RAVI_TI_STRING,
-  RAVI_TI_FUNCTION,
-  RAVI_TI_USERDATA,
-  RAVI_TI_OTHER
-} ravi_type_index;
-
-typedef uint32_t ravi_type_map;
-
-#define RAVI_TM_NIL (((ravi_type_map)1)<<RAVI_TI_NIL)
-#define RAVI_TM_FALSE (((ravi_type_map)1)<<RAVI_TI_FALSE)
-#define RAVI_TM_TRUE (((ravi_type_map)1)<<RAVI_TI_TRUE)
-#define RAVI_TM_INTEGER (((ravi_type_map)1)<<RAVI_TI_INTEGER)
-#define RAVI_TM_FLOAT (((ravi_type_map)1)<<RAVI_TI_FLOAT)
-#define RAVI_TM_INTEGER_ARRAY (((ravi_type_map)1)<<RAVI_TI_INTEGER_ARRAY)
-#define RAVI_TM_FLOAT_ARRAY (((ravi_type_map)1)<<RAVI_TI_FLOAT_ARRAY)
-#define RAVI_TM_TABLE (((ravi_type_map)1)<<RAVI_TI_TABLE)
-#define RAVI_TM_STRING (((ravi_type_map)1)<<RAVI_TI_STRING)
-#define RAVI_TM_FUNCTION (((ravi_type_map)1)<<RAVI_TI_FUNCTION)
-#define RAVI_TM_USERDATA (((ravi_type_map)1)<<RAVI_TI_USERDATA)
-#define RAVI_TM_OTHER (((ravi_type_map)1)<<RAVI_TI_OTHER)
-
-#define RAVI_TM_FALSISH (RAVI_TM_NIL | RAVI_TM_FALSE)
-#define RAVI_TM_TRUISH (~RAVI_TM_FALSISH)
-#define RAVI_TM_BOOLEAN (RAVI_TM_FALSE | RAVI_TM_TRUE)
-#define RAVI_TM_NUMBER (RAVI_TM_INTEGER | RAVI_TM_FLOAT)
-#define RAVI_TM_STRING_OR_NIL (RAVI_TM_STRING | RAVI_TM_NIL)
-#define RAVI_TM_FUNCTION_OR_NIL (RAVI_TM_FUNCTION | RAVI_TM_NIL)
-#define RAVI_TM_BOOLEAN_OR_NIL (RAVI_TM_BOOLEAN | RAVI_TM_NIL)
-#define RAVI_TM_USERDATA_OR_NIL (RAVI_TM_USERDATA | RAVI_TM_NIL)
-#define RAVI_TM_INTEGER_OR_NIL (RAVI_TM_INTEGER | RAVI_TM_NIL)
-#define RAVI_TM_FLOAT_OR_NIL (RAVI_TM_FLOAT | RAVI_TM_NIL)
-#define RAVI_TM_TABLE_OR_NIL (RAVI_TM_TABLE | RAVI_TM_NIL)
-#define RAVI_TM_INDEXABLE (RAVI_TM_INTEGER_ARRAY | RAVI_TM_FLOAT_ARRAY | RAVI_TM_TABLE)
-#define RAVI_TM_ANY (~0)
 
-typedef enum {
-  RAVI_TNIL = RAVI_TM_NIL,        /* NIL */
-  RAVI_TNUMINT = RAVI_TM_INTEGER, /* integer number */
-  RAVI_TNUMFLT = RAVI_TM_FLOAT,   /* floating point number */
-  RAVI_TNUMBER = RAVI_TM_NUMBER,
-  RAVI_TARRAYINT = RAVI_TM_INTEGER_ARRAY,   /* array of ints */
-  RAVI_TARRAYFLT = RAVI_TM_FLOAT_ARRAY,     /* array of doubles */
-  RAVI_TTABLE = RAVI_TM_TABLE,              /* Lua table */
-  RAVI_TSTRING = RAVI_TM_STRING,     /* string */
-  RAVI_TFUNCTION = RAVI_TM_FUNCTION, /* Lua or C Function */
-  RAVI_TBOOLEAN = RAVI_TM_BOOLEAN,   /* boolean */
-  RAVI_TTRUE = RAVI_TM_TRUE,
-  RAVI_TFALSE = RAVI_TM_FALSE,
-  RAVI_TUSERDATA = RAVI_TM_USERDATA, /* userdata or lightuserdata */
-  RAVI_TANY = RAVI_TM_ANY,                  /* Lua dynamic type */
-} ravitype_t;
 
 /*
 ** Tagged Values. This is the basic representation of values in Lua,
@@ -125,7 +114,7 @@ typedef enum {
 ** Union of all Lua values
 */
 typedef union Value {
-  struct GCObject *gc;    /* collectable objects */
+  GCObject *gc;    /* collectable objects */
   void *p;         /* light userdata */
   int b;           /* booleans */
   lua_CFunction f; /* light C functions */
@@ -134,19 +123,22 @@ typedef union Value {
 } Value;
 
 
-/*
-** Tagged Values. This is the basic representation of values in Lua:
-** an actual value plus a tag with its type.
-*/
-
 #define TValuefields	Value value_; LuaType tt_
 
+
 typedef struct lua_TValue {
   TValuefields;
 } TValue;
 
+
+
+/* macro defining a nil value */
+#define NILCONSTANT	{NULL}, LUA_TNIL
+
+
 #define val_(o)		((o)->value_)
 
+
 /* raw type tag of a TValue */
 #define rttype(o)	((o)->tt_)
 
@@ -164,196 +156,198 @@ typedef struct lua_TValue {
 /* Macros to test type */
 #define checktag(o,t)		(rttype(o) == (t))
 #define checktype(o,t)		(ttnov(o) == (t))
+#define ttisnumber(o)		checktype((o), LUA_TNUMBER)
+#define ttisfloat(o)		checktag((o), LUA_TNUMFLT)
+#define ttisinteger(o)		checktag((o), LUA_TNUMINT)
+#define ttisnil(o)		checktag((o), LUA_TNIL)
+#define ttisboolean(o)		checktag((o), LUA_TBOOLEAN)
+#define ttislightuserdata(o)	checktag((o), LUA_TLIGHTUSERDATA)
+#define ttisstring(o)		checktype((o), LUA_TSTRING)
+#define ttisshrstring(o)	checktag((o), ctb(LUA_TSHRSTR))
+#define ttislngstring(o)	checktag((o), ctb(LUA_TLNGSTR))
+/* RAVI change: we support two sub types of table type
+   and hence need to distinguish between the types.
+   ttistable() returns true for all table types
+   ttisLtable() only returns true if the value is a Lua table 
+   ttisiarray() only returns true if the value is a Ravi subtype integer[]
+   ttisfarray() only returns true if the value is a Ravi subtype number[]
+ */
+#define ttistable(o)		checktype((o), LUA_TTABLE)
+#define ttisiarray(o)    checktag((o), ctb(RAVI_TIARRAY))
+#define ttisfarray(o)    checktag((o), ctb(RAVI_TFARRAY))
+#define ttisLtable(o)    checktag((o), ctb(LUA_TTABLE))
+#define ttisfunction(o)		checktype(o, LUA_TFUNCTION)
+#define ttisclosure(o)		((rttype(o) & 0x1F) == LUA_TFUNCTION)
+#define ttisCclosure(o)		checktag((o), ctb(LUA_TCCL))
+#define ttisLclosure(o)		checktag((o), ctb(LUA_TLCL))
+#define ttislcf(o)		checktag((o), LUA_TLCF)
+#define ttisfcf(o)      (ttype(o) == RAVI_TFCF)
+#define ttisfulluserdata(o)	checktag((o), ctb(LUA_TUSERDATA))
+#define ttisthread(o)		checktag((o), ctb(LUA_TTHREAD))
+#define ttisdeadkey(o)		checktag((o), LUA_TDEADKEY)
 
 
-/* Macros for internal tests */
+/* Macros to access values */
+#define ivalue(o)	check_exp(ttisinteger(o), val_(o).i)
+#define fltvalue(o)	check_exp(ttisfloat(o), val_(o).n)
+#define nvalue(o)	check_exp(ttisnumber(o), \
+	(ttisinteger(o) ? cast_num(ivalue(o)) : fltvalue(o)))
+#define gcvalue(o)	check_exp(iscollectable(o), val_(o).gc)
+#define pvalue(o)	check_exp(ttislightuserdata(o), val_(o).p)
+#define tsvalue(o)	check_exp(ttisstring(o), gco2ts(val_(o).gc))
+#define uvalue(o)	check_exp(ttisfulluserdata(o), gco2u(val_(o).gc))
+#define clvalue(o)	check_exp(ttisclosure(o), gco2cl(val_(o).gc))
+#define clLvalue(o)	check_exp(ttisLclosure(o), gco2lcl(val_(o).gc))
+#define clCvalue(o)	check_exp(ttisCclosure(o), gco2ccl(val_(o).gc))
+#define fvalue(o)	check_exp(ttislcf(o), val_(o).f)
+#define fcfvalue(o) check_exp(ttisfcf(o), val_(o).p)
+#define hvalue(o)	check_exp(ttistable(o), gco2t(val_(o).gc))
+#define bvalue(o)	check_exp(ttisboolean(o), val_(o).b)
+#define thvalue(o)	check_exp(ttisthread(o), gco2th(val_(o).gc))
+/* a dead value may get the 'gc' field, but cannot access its contents */
+#define deadvalue(o)	check_exp(ttisdeadkey(o), cast(void *, val_(o).gc))
+
+#define l_isfalse(o)	(ttisnil(o) || (ttisboolean(o) && bvalue(o) == 0))
+
 
-/* collectable object has the same tag as the original value */
+#define iscollectable(o)	(rttype(o) & BIT_ISCOLLECTABLE)
+
+
+/* Macros for internal tests */
 #define righttt(obj)		(ttype(obj) == gcvalue(obj)->tt)
 
-/*
-** Any value being manipulated by the program either is non
-** collectable, or the collectable object has the right tag
-** and it is not dead.
-*/
 #define checkliveness(L,obj) \
 	lua_longassert(!iscollectable(obj) || \
 		(righttt(obj) && (L == NULL || !isdead(G(L),gcvalue(obj)))))
 
 
 /* Macros to set values */
-
-/* set a value's tag */
 #define settt_(o,t)	((o)->tt_=(t))
 
+#define setfltvalue(obj,x) \
+  { TValue *io=(obj); val_(io).n=(x); settt_(io, LUA_TNUMFLT); }
 
-/* main macro to copy values (from 'obj1' to 'obj2') */
-#define setobj(L,obj1,obj2) \
-	{ TValue *io1=(obj1); const TValue *io2=(obj2); \
-          io1->value_ = io2->value_; settt_(io1, io2->tt_); \
-	  (void)L; checkliveness(L,io1); }
-
-
-/*
-** different types of assignments, according to destination
-*/
-
-/* from stack to (same) stack */
-#define setobjs2s	setobj
-/* to stack (not from same stack) */
-#define setobj2s	setobj
-/* from table to same table */
-#define setobjt2t	setobj
-/* to new object */
-#define setobj2n	setobj
-/* to table */
-#define setobj2t	setobj
+#define chgfltvalue(obj,x) \
+  { TValue *io=(obj); lua_assert(ttisfloat(io)); val_(io).n=(x); }
 
-typedef TValue *StkId;  /* index to stack elements */
-/*
-** {==================================================================
-** Nil
-** ===================================================================
-*/
+#define setivalue(obj,x) \
+  { TValue *io=(obj); val_(io).i=(x); settt_(io, LUA_TNUMINT); }
 
-/* macro defining a nil value */
-#define NILCONSTANT	{NULL}, LUA_TNIL
-/* macro to test for (any kind of) nil */
-#define ttisnil(o)		checktag((o), LUA_TNIL)
+#define chgivalue(obj,x) \
+  { TValue *io=(obj); lua_assert(ttisinteger(io)); val_(io).i=(x); }
 
 #define setnilvalue(obj) settt_(obj, LUA_TNIL)
-/* }================================================================== */
-
 
-/*
-** {==================================================================
-** Booleans
-** ===================================================================
-*/
+#define setfvalue(obj,x) \
+  { TValue *io=(obj); val_(io).f=(x); settt_(io, LUA_TLCF); }
 
+/* The Fast C function call type is encoded as two
+   bytes. The Hi Byte holds a function tag. The Lo Byte
+   holds the Lua typecode */
+#define setfvalue_fastcall(obj, x, tag) \
+  {                       \
+    TValue *io = (obj);   \
+    lua_assert(tag >= 1 && tag < 0x80); \
+    val_(io).p = (x);     \
+    settt_(io, ((tag << 8) | RAVI_TFCF)); \
+  }
+#define getfcf_tag(typecode) (typecode >> 8)
 
-#define ttisboolean(o)		checktag((o), LUA_TBOOLEAN)
-#define bvalue(o)	check_exp(ttisboolean(o), val_(o).b)
-#define l_isfalse(o)	(ttisnil(o) || (ttisboolean(o) && bvalue(o) == 0))
+#define setpvalue(obj,x) \
+  { TValue *io=(obj); val_(io).p=(x); settt_(io, LUA_TLIGHTUSERDATA); }
 
 #define setbvalue(obj,x) \
   { TValue *io=(obj); val_(io).b=(x); settt_(io, LUA_TBOOLEAN); }
-/* }================================================================== */
 
+#define setgcovalue(L,obj,x) \
+  { TValue *io = (obj); GCObject *i_g=(x); \
+    val_(io).gc = i_g; settt_(io, ctb(i_g->tt)); }
 
-/*
-** {==================================================================
-** Threads
-** ===================================================================
-*/
+#define setsvalue(L,obj,x) \
+  { TValue *io = (obj); TString *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(x_->tt)); \
+    checkliveness(L,io); }
 
-#define ttisthread(o)		checktag((o), ctb(LUA_TTHREAD))
-#define thvalue(o)	check_exp(ttisthread(o), gco2th(val_(o).gc))
+#define setuvalue(L,obj,x) \
+  { TValue *io = (obj); Udata *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TUSERDATA)); \
+    checkliveness(L,io); }
 
 #define setthvalue(L,obj,x) \
   { TValue *io = (obj); lua_State *x_ = (x); \
     val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TTHREAD)); \
     checkliveness(L,io); }
 
+#define setclLvalue(L,obj,x) \
+  { TValue *io = (obj); LClosure *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TLCL)); \
+    checkliveness(L,io); }
 
-/* }================================================================== */
-
-
-/*
-** {==================================================================
-** Collectable Objects
-** ===================================================================
-*/
-
-/*
-** Common Header for all collectable objects (in macro form, to be
-** included in other objects)
-** Note that tt field is a byte.
-*/
-#define CommonHeader	struct GCObject *next; lu_byte tt; lu_byte marked
-
-
-/* Common type for all collectable objects */
-typedef struct GCObject {
-  CommonHeader;
-} GCObject;
+#define setclCvalue(L,obj,x) \
+  { TValue *io = (obj); CClosure *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TCCL)); \
+    checkliveness(L,io); }
 
+#define sethvalue(L,obj,x) \
+  { TValue *io = (obj); Table *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TTABLE)); \
+    checkliveness(L,io); }
 
-/* Bit mark for collectable types */
-#define BIT_ISCOLLECTABLE	(1 << 15)
+/** RAVI extension **/
+#define setiarrayvalue(L,obj,x) \
+  { TValue *io = (obj); Table *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(RAVI_TIARRAY)); \
+    checkliveness(L,io); }
 
-#define iscollectable(o)	(rttype(o) & BIT_ISCOLLECTABLE)
+/** RAVI extension **/
+#define setfarrayvalue(L,obj,x) \
+  { TValue *io = (obj); Table *x_ = (x); \
+    val_(io).gc = obj2gco(x_); settt_(io, ctb(RAVI_TFARRAY)); \
+    checkliveness(L,io); }
 
-/* mark a tag as collectable */
-#define ctb(t)			((t) | BIT_ISCOLLECTABLE)
+#define setdeadvalue(obj)	settt_(obj, LUA_TDEADKEY)
 
-#define gcvalue(o)	check_exp(iscollectable(o), val_(o).gc)
 
-#define setgcovalue(L,obj,x) \
-  { TValue *io = (obj); GCObject *i_g=(x); \
-    val_(io).gc = i_g; settt_(io, ctb(i_g->tt)); }
 
-/* }================================================================== */
+#define setobj(L,obj1,obj2) \
+	{ TValue *io1=(obj1); const TValue *io2=(obj2); \
+          io1->value_ = io2->value_; io1->tt_ = io2->tt_; \
+	  (void)L; checkliveness(L,io1); }
 
 
 /*
-** {==================================================================
-** Numbers
-** ===================================================================
+** different types of assignments, according to destination
 */
 
-/* Variant tags for numbers */
-#define LUA_TNUMFLT	makevariant(LUA_TNUMBER, 0)  /* float numbers */
-#define LUA_TNUMINT	makevariant(LUA_TNUMBER, 1)  /* integer numbers */
-#define ttisnumber(o)		checktype((o), LUA_TNUMBER)
-#define ttisfloat(o)		checktag((o), LUA_TNUMFLT)
-#define ttisinteger(o)		checktag((o), LUA_TNUMINT)
-
-#define nvalue(o)	check_exp(ttisnumber(o), \
-	(ttisinteger(o) ? cast_num(ivalue(o)) : fltvalue(o)))
-#define fltvalue(o)	check_exp(ttisfloat(o), val_(o).n)
-#define ivalue(o)	check_exp(ttisinteger(o), val_(o).i)
-
-#define setfltvalue(obj,x) \
-  { TValue *io=(obj); val_(io).n=(x); settt_(io, LUA_TNUMFLT); }
-
-#define chgfltvalue(obj,x) \
-  { TValue *io=(obj); lua_assert(ttisfloat(io)); val_(io).n=(x); }
-
-#define setivalue(obj,x) \
-  { TValue *io=(obj); val_(io).i=(x); settt_(io, LUA_TNUMINT); }
+/* from stack to (same) stack */
+#define setobjs2s	setobj
+/* to stack (not from same stack) */
+#define setobj2s	setobj
+#define setsvalue2s	setsvalue
+#define sethvalue2s	sethvalue
+#define setptvalue2s	setptvalue
+/* from table to same table */
+#define setobjt2t	setobj
+/* to new object */
+#define setobj2n	setobj
+#define setsvalue2n	setsvalue
+/* to table */
+#define setobj2t	setobj
 
-#define chgivalue(obj,x) \
-  { TValue *io=(obj); lua_assert(ttisinteger(io)); val_(io).i=(x); }
 
-/* }================================================================== */
 
 
 /*
-** {==================================================================
-** Strings
-** ===================================================================
+** {======================================================
+** types and prototypes
+** =======================================================
 */
 
-/* Variant tags for strings */
-#define LUA_TSHRSTR	makevariant(LUA_TSTRING, 0)  /* short strings */
-#define LUA_TLNGSTR	makevariant(LUA_TSTRING, 1)  /* long strings */
 
-#define ttisstring(o)		checktype((o), LUA_TSTRING)
-#define ttisshrstring(o)	checktag((o), ctb(LUA_TSHRSTR))
-#define ttislngstring(o)	checktag((o), ctb(LUA_TLNGSTR))
+typedef TValue *StkId;  /* index to stack elements */
 
-#define tsvalue(o)	check_exp(ttisstring(o), gco2ts(val_(o).gc))
 
-#define setsvalue(L,obj,x) \
-  { TValue *io = (obj); TString *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(x_->tt)); \
-    checkliveness(L,io); }
 
-/* set a string to the stack */
-#define setsvalue2s	setsvalue
-/* set a string to a new object */
-#define setsvalue2n	setsvalue
 
 /*
 ** Header for string value; string bytes follow the end of this structure
@@ -379,6 +373,7 @@ typedef union UTString {
   TString tsv;
 } UTString;
 
+
 /*
 ** Get the actual string (array of bytes) from a 'TString'.
 ** (Access to 'extra' ensures that value is really a 'TString'.)
@@ -396,34 +391,6 @@ typedef union UTString {
 /* get string length from 'TValue *o' */
 #define vslen(o)	tsslen(tsvalue(o))
 
-/* }================================================================== */
-
-
-/*
-** {==================================================================
-** Userdata
-** ===================================================================
-*/
-
-
-/*
-** Light userdata should be a variant of userdata, but for compatibility
-** reasons they are also different types.
-*/
-
-#define ttislightuserdata(o)	checktag((o), LUA_TLIGHTUSERDATA)
-#define ttisfulluserdata(o)	checktag((o), ctb(LUA_TUSERDATA))
-
-#define pvalue(o)	check_exp(ttislightuserdata(o), val_(o).p)
-#define uvalue(o)	check_exp(ttisfulluserdata(o), gco2u(val_(o).gc))
-
-#define setpvalue(obj,x) \
-  { TValue *io=(obj); val_(io).p=(x); settt_(io, LUA_TLIGHTUSERDATA); }
-
-#define setuvalue(L,obj,x) \
-  { TValue *io = (obj); Udata *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TUSERDATA)); \
-    checkliveness(L,io); }
 
 /*
 ** Header for userdata; memory area follows the end of this structure
@@ -465,14 +432,25 @@ typedef union UUdata {
 	  io->value_ = iu->user_; settt_(io, iu->ttuv_); \
 	  checkliveness(L,io); }
 
-/* }================================================================== */
-
-
-/*
-** {==================================================================
-** Prototypes
-** ===================================================================
-*/
+/* RAVI: Following are the types we will use
+** use in parsing. The rationale for types is
+** performance - as of now these are the only types that
+** we care about from a performance point of view - if any
+** other types appear then they are all treated as ANY
+**/
+typedef enum {
+  RAVI_TANY = 0,      /* Lua dynamic type */
+  RAVI_TNUMINT = 1,    /* integer number */
+  RAVI_TNUMFLT,        /* floating point number */
+  RAVI_TARRAYINT,      /* array of ints */
+  RAVI_TARRAYFLT,      /* array of doubles */
+  RAVI_TFUNCTION,      /* Lua or C Function */
+  RAVI_TTABLE,         /* Lua table */
+  RAVI_TSTRING,        /* string */
+  RAVI_TNIL,           /* NIL */
+  RAVI_TBOOLEAN,       /* boolean */
+  RAVI_TUSERDATA       /* userdata or lightuserdata */
+} ravitype_t;
 
 /*
 ** Description of an upvalue for function prototypes
@@ -480,7 +458,7 @@ typedef union UUdata {
 typedef struct Upvaldesc {
   TString *name;  /* upvalue name (for debug information) */
   TString *usertype; /* RAVI extension: name of user type */
-  ravi_type_map ravi_type_map; /* RAVI type of upvalue */
+  lu_byte ravi_type; /* RAVI type of upvalue */
   lu_byte instack;  /* whether it is in stack (register) */
   lu_byte idx;  /* index of upvalue (in stack or in outer function's list) */
 } Upvaldesc;
@@ -495,7 +473,7 @@ typedef struct LocVar {
   TString *usertype; /* RAVI extension: name of user type */
   int startpc;  /* first point where variable is active */
   int endpc;    /* first point where variable is dead */
-  ravi_type_map ravi_type_map; /* RAVI type of the variable - RAVI_TANY if unknown */
+  lu_byte ravi_type; /* RAVI type of the variable - RAVI_TANY if unknown */
 } LocVar;
 
 /** RAVI changes start */
@@ -524,7 +502,7 @@ typedef struct RaviJITProto {
 */
 typedef struct Proto {
   CommonHeader;
-  lu_byte numparams;  /* number of fixed (named) parameters */
+  lu_byte numparams;  /* number of fixed parameters */
   lu_byte is_vararg;
   lu_byte maxstacksize;  /* number of registers needed by this function */
   int sizeupvalues;  /* size of 'upvalues' */
@@ -548,119 +526,18 @@ typedef struct Proto {
   RaviJITProto ravi_jit;
 } Proto;
 
-/*
-** {==================================================================
-** Closures
-** ===================================================================
-*/
 
 
 /*
-** LUA_TFUNCTION variants:
-** 0 - Lua function
-** 1 - light C function
-** 2 - regular C function (closure)
-** 4 - fast light C dunction (Ravi extension)
+** Lua Upvalues
 */
+typedef struct UpVal UpVal;
 
-/* Variant tags for functions */
-#define LUA_TLCL	makevariant(LUA_TFUNCTION, 0)  /* Lua closure */
-#define LUA_TLCF	makevariant(LUA_TFUNCTION, 1)  /* light C function */
-#define LUA_TCCL	makevariant(LUA_TFUNCTION, 2)  /* C closure */
-#define RAVI_TFCF	makevariant(LUA_TFUNCTION, 4)  /* Ravi extension: fast light C function */
-
-#define ttisfunction(o)		checktype(o, LUA_TFUNCTION)
-#define ttisclosure(o)		((rttype(o) & 0x1F) == LUA_TFUNCTION)
-#define ttisLclosure(o)		checktag((o), ctb(LUA_TLCL))
-#define ttislcf(o)		checktag((o), LUA_TLCF)
-
-#define ttisCclosure(o)		checktag((o), ctb(LUA_TCCL))
-#define ttisfcf(o)      (ttype(o) == RAVI_TFCF)
-#define isLfunction(o)	ttisLclosure(o)
-
-#define clvalue(o)	check_exp(ttisclosure(o), gco2cl(val_(o).gc))
-#define clLvalue(o)	check_exp(ttisLclosure(o), gco2lcl(val_(o).gc))
-#define fvalue(o)	check_exp(ttislcf(o), val_(o).f)
-#define clCvalue(o)	check_exp(ttisCclosure(o), gco2ccl(val_(o).gc))
-#define fcfvalue(o) check_exp(ttisfcf(o), val_(o).p)
-
-#define setclLvalue(L,obj,x) \
-  { TValue *io = (obj); LClosure *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TLCL)); \
-    checkliveness(L,io); }
-
-#define setfvalue(obj,x) \
-  { TValue *io=(obj); val_(io).f=(x); settt_(io, LUA_TLCF); }
-
-#define setclCvalue(L,obj,x) \
-  { TValue *io = (obj); CClosure *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TCCL)); \
-    checkliveness(L,io); }
-	
-
-/* The Fast C function call type is encoded as two
-   bytes. The Hi Byte holds a function tag. The Lo Byte
-   holds the Lua typecode */
-#define setfvalue_fastcall(obj, x, tag) \
-  {                       \
-    TValue *io = (obj);   \
-    lua_assert(tag >= 1 && tag < 0x80); \
-    val_(io).p = (x);     \
-    settt_(io, ((tag << 8) | RAVI_TFCF)); \
-  }
-#define getfcf_tag(typecode) (typecode >> 8)
 
 /*
-* Upvalues for Lua closures. The UpVal structure mediates the connection between a
-* closure and a variable. An upvalue may be two states: open or closed.
-* When the upvalue is created, it is open, and its pointer points to the corresponding
-* variable in the Lua stack. That is, an open upvalue is one that's v is pointing to
-* the stack. When the upvalue is closed, the value is moved from the stack to the
-* UpVal structure itself (value) and the pointer v is corrected to point internally.
-*
-* At any point a variable can have at most one upvalue pointing to it, and all
-* closures that reference the upvalue access this shared  upvalue. Lua keeps a
-* linked list of open upvalues of a stack. This list is ordered by the level of the
-* corresponding variables on the stack. When Lua needs an upvalue for a local variable
-* it traverse this linked list. If it finds an upvalue for the variable it reuses it
-* thus ensuring that closures share the same upvalue.
-*
-* Because the list is ordered and there is at most one upvalue for each variable
-* the maximum number of elements to be traversed when looking for a variable in this
-* list can be known at compile time. This maximum is the number of variables that escape
-* to inner closures and that are declared between the closure and the external variable.
-* For instance
-*
-*    function foo()
-*      local a, b, c, d
-*      local f1 = function() return d + b end
-*      local f2 = function() return f() + a end
-*
-* When Lua instantiates f2 it will traverse exactly three upvalues before realizing
-* that a has no upvalue yet: f1, d, and b in that order.
-*
-* When a variable goes out of scope, its corrsponding update (if there is one) must
-* be closed. The list of open upvalues is also used for this task. When compiling a
-* block that contains variables that escape, a "close" operation must be emitted (in Ravi
-* there is no explicit close op, the JMP instruction takes care of it) to close all
-* upvalues up to this level, at the end of the block.
+** Closures
 */
 
-typedef struct UpVal {
-  TValue *v;  /* points to stack or to its own value */
-  unsigned int refcount;  /* reference counter */
-  unsigned int flags; /* Used to mark deferred values */
-  union {
-    struct {  /* (when open) */
-      struct UpVal *next;  /* linked list */
-      int touched;  /* mark to avoid cycles with dead threads */
-    } open;
-    TValue value;  /* the value (when closed) */
-  } u;
-} UpVal;
-
-
-
 #define ClosureHeader \
 	CommonHeader; lu_byte nupvalues; GCObject *gclist
 
@@ -674,7 +551,7 @@ typedef struct CClosure {
 typedef struct LClosure {
   ClosureHeader;
   struct Proto *p;
-  UpVal *upvals[1];  /* list of upvalues - each upvalue represents one non-local variable used by the closure */
+  UpVal *upvals[1];  /* list of upvalues */
 } LClosure;
 
 
@@ -684,64 +561,15 @@ typedef union Closure {
 } Closure;
 
 
-#define getproto(o)	(clLvalue(o)->p)
+#define isLfunction(o)	ttisLclosure(o)
 
-/* }================================================================== */
+#define getproto(o)	(clLvalue(o)->p)
 
 
 /*
-** {==================================================================
 ** Tables
-** ===================================================================
 */
 
-/* RAVI change: we support two sub types of table type
-   and hence need to distinguish between the types.
-   ttistable() returns true for all table types
-   ttisLtable() only returns true if the value is a Lua table 
-   ttisiarray() only returns true if the value is a Ravi subtype integer[]
-   ttisfarray() only returns true if the value is a Ravi subtype number[]
- */
-/** RAVI table subtypes **/
-#define RAVI_TIARRAY makevariant(LUA_TTABLE, 1)  /* Ravi int array */
-#define RAVI_TFARRAY makevariant(LUA_TTABLE, 2)  /* Ravi float array */
-
-#define ttistable(o)     checktype((o), LUA_TTABLE)
-#define ttisiarray(o)    checktag((o), ctb(RAVI_TIARRAY))
-#define ttisfarray(o)    checktag((o), ctb(RAVI_TFARRAY))
-#define ttisarray(o)     (ttisiarray(o) || ttisfarray(o))
-#define ttisLtable(o)    checktag((o), ctb(LUA_TTABLE))
-
-#define ttisdeadkey(o)		checktag((o), LUA_TDEADKEY)
-
-/* Macros to access values */
-
-#define hvalue(o)	check_exp(ttisLtable(o), gco2t(val_(o).gc))
-#define arrvalue(o) check_exp(ttisarray(o), gco2array(val_(o).gc))
-/* a dead value may get the 'gc' field, but cannot access its contents */
-#define deadvalue(o)	check_exp(ttisdeadkey(o), cast(void *, val_(o).gc))
-
-#define sethvalue(L,obj,x) \
-  { TValue *io = (obj); Table *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(LUA_TTABLE)); \
-    checkliveness(L,io); }
-
-/** RAVI extension **/
-#define setiarrayvalue(L,obj,x) \
-  { TValue *io = (obj); RaviArray *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(RAVI_TIARRAY)); \
-    checkliveness(L,io); }
-
-/** RAVI extension **/
-#define setfarrayvalue(L,obj,x) \
-  { TValue *io = (obj); RaviArray *x_ = (x); \
-    val_(io).gc = obj2gco(x_); settt_(io, ctb(RAVI_TFARRAY)); \
-    checkliveness(L,io); }
-
-#define sethvalue2s	sethvalue
-#define setptvalue2s	setptvalue
-
-
 typedef union TKey {
   struct {
     TValuefields;
@@ -751,6 +579,11 @@ typedef union TKey {
 } TKey;
 
 
+/* copy a value into a key without messing up field 'next' */
+#define setnodekey(L,key,obj) \
+	{ TKey *k_=(key); const TValue *io_=(obj); \
+	  k_->nk.value_ = io_->value_; k_->nk.tt_ = io_->tt_; \
+	  (void)L; checkliveness(L,io_); }
 
 
 typedef struct Node {
@@ -758,11 +591,20 @@ typedef struct Node {
   TKey i_key;
 } Node;
 
-/* copy a value into a key without messing up field 'next' */
-#define setnodekey(L,key,obj) \
-	{ TKey *k_=(key); const TValue *io_=(obj); \
-	  k_->nk.value_ = io_->value_; k_->nk.tt_ = io_->tt_; \
-	  (void)L; checkliveness(L,io_); }
+/** RAVI extension */
+typedef enum RaviArrayModifer {
+  RAVI_ARRAY_SLICE = 1,
+  RAVI_ARRAY_FIXEDSIZE = 2
+} RaviArrayModifier;
+
+/** RAVI extension */
+typedef struct RaviArray {
+  char *data;             /* Note that the array data is 0-based so this holds 1+Lua length items */
+  unsigned int len;       /* RAVI len specialization, holds real length which is 1+Lua length */
+  unsigned int size;      /* amount of memory allocated */
+  lu_byte array_type;     /* RAVI specialization */
+  lu_byte array_modifier; /* Flags that affect how the array is handled */
+} RaviArray;
 
 typedef struct Table {
   CommonHeader;
@@ -774,74 +616,15 @@ typedef struct Table {
   Node *lastfree;  /* any free position is before this position */
   struct Table *metatable;
   GCObject *gclist;
+  /** RAVI extension */
+  RaviArray ravi_array;
 #if RAVI_USE_NEWHASH
   // TODO we should reorganize this structure
   unsigned int hmask; /* Hash part mask (size of hash part - 1) - borrowed from LuaJIT */
 #endif
 } Table;
 
-/*
-** Macros to manipulate keys inserted in nodes
-*/
-#define keytt(node)         ((node)->i_key.nk.tt_)
-#define keyval(node)        ((node)->i_key.nk.value_)
-
-#define keyisnil(node)      (keytt(node) == LUA_TNIL)
-#define keyisinteger(node)  (keytt(node) == LUA_TNUMINT)
-#define keyival(node)       (keyval(node).i)
-#define keyfltval(node)     (keyval(node).n)
-#define keyfval(node)       (keyval(node).f)
-#define keypval(node)       (keyval(node).p)
-#define keybval(node)       (keyval(node).b)
-#define keygcval(node)      (keyval(node).gc)
-#define keyisshrstr(node)   (keytt(node) == ctb(LUA_TSHRSTR))
-#define keystrval(node)     (gco2ts(keyval(node).gc))
-
-#define setnilkey(node)		(keytt(node) = LUA_TNIL)
-
-#define keyiscollectable(n)	(keytt(n) & BIT_ISCOLLECTABLE)
-
-#define gckey(n)	(keyval(n).gc)
-#define gckeyN(n)	(keyiscollectable(n) ? gckey(n) : NULL)
-
-#define setdeadvalue(obj)	settt_(obj, LUA_TDEADKEY)
-
-/* }================================================================== */
-
-/** RAVI extension */
-typedef enum RaviArrayModifer {
-  RAVI_ARRAY_SLICE = 1, /* Array is a slice - implies fixed size */
-  RAVI_ARRAY_FIXEDSIZE = 2, /* Fixed size array */
-  RAVI_ARRAY_ALLOCATED = 4, /* Array has memory allocated - cannot be true for slices */
-  RAVI_ARRAY_ISFLOAT = 8 /* A number array */
-} RaviArrayModifier;
-
-enum {
-  RAVI_ARRAY_MAX_INLINE = 3 /* By default we allow for inline storage of 3 elements */,
-};
-
-/** RAVI extension */
-/* Array types look like Userdata from GC point of view, but
- * have the same base type as Lua tables.
- */
-typedef struct RaviArray {
-  CommonHeader;
-  lu_byte flags;
-  unsigned int len;  /* array length, holds real length which is 1+Lua length */
-  unsigned int size; /* size of data, in arrays (not slices) if == RAVI_ARRAY_MAX_INLINE then it means we are using inline storage */
-  union {
-    lua_Number numarray[RAVI_ARRAY_MAX_INLINE];
-    lua_Integer intarray[RAVI_ARRAY_MAX_INLINE];
-    struct RaviArray* parent; /* Only set if this is a slice, parent must be a slice or a fixed length array */
-  };
-  char *data;    /* Pointer to data. In case of slices points in parent->data. In case of arrays this may point to heap or internal data */
-  struct Table *metatable;
-} RaviArray;
 
-#define getsliceunderlying(L,s,o) \
-	{ TValue *io=(o); const RaviArray *is = (s); \
-	  io->value_.gc = obj2gco(is->parent); settt_(io, ctb(is->parent->tt)); \
-	  checkliveness(L,io); }
 
 /*
 ** 'module' operation for hashing (size is always a power of 2)
@@ -876,11 +659,8 @@ LUAI_FUNC void luaO_tostring (lua_State *L, StkId obj);
 LUAI_FUNC const char *luaO_pushvfstring (lua_State *L, const char *fmt,
                                                        va_list argp);
 LUAI_FUNC const char *luaO_pushfstring (lua_State *L, const char *fmt, ...);
-LUAI_FUNC void luaO_chunkid (char *out, const char *source, size_t srclen);
-
-
+LUAI_FUNC void luaO_chunkid (char *out, const char *source, size_t len);
 
-LUAI_FUNC int ravi_checktype(lua_State *L, StkId input, ravi_type_map type, TString* usertype);
 
 #endif
 

include/lopcodes.h

@@ -219,23 +219,13 @@ OP_RAVI_DIVII,  /*	A B C	R(A) := RK(B) / RK(C)		*/
 
 OP_RAVI_TOINT, /* A R(A) := toint(R(A)) */
 OP_RAVI_TOFLT, /* A R(A) := tofloat(R(A)) */
+OP_RAVI_TOIARRAY, /* A R(A) := to_arrayi(R(A)) */
+OP_RAVI_TOFARRAY, /* A R(A) := to_arrayf(R(A)) */
 OP_RAVI_TOTAB,    /* A R(A) := to_table(R(A)) */
 OP_RAVI_TOSTRING, /* A R(A) := assert_string(R(A)) */
-OP_RAVI_TOBOOLEAN, /* A R(A) := assert_string(R(A)) */
 OP_RAVI_TOCLOSURE, /* A R(A) := assert_closure(R(A)) */
 OP_RAVI_TOTYPE,  /* A R(A) := assert_usertype(R(A)), where usertype has metatable in Lua registry */
 
-OP_RAVI_TOINT_NIL, /* A R(A) := toint(R(A)) */
-OP_RAVI_TOFLT_NIL, /* A R(A) := tofloat(R(A)) */
-OP_RAVI_TOTAB_NIL,    /* A R(A) := to_table(R(A)) */
-OP_RAVI_TOSTRING_NIL, /* A R(A) := assert_string(R(A)) */
-OP_RAVI_TOBOOLEAN_NIL, /* A R(A) := assert_string(R(A)) */
-OP_RAVI_TOCLOSURE_NIL, /* A R(A) := assert_closure(R(A)) */
-OP_RAVI_TOTYPE_NIL,  /* A R(A) := assert_usertype(R(A)), where usertype has metatable in Lua registry */
-
-OP_RAVI_TOIARRAY, /* A R(A) := to_arrayi(R(A)) */
-OP_RAVI_TOFARRAY, /* A R(A) := to_arrayf(R(A)) */
-
 OP_RAVI_MOVEI,       /*	A B	R(A) := R(B), check R(B) is int	*/
 OP_RAVI_MOVEF,       /*	A B	R(A) := R(B), check R(B) is float */
 OP_RAVI_MOVEIARRAY,  /* A B R(A) := R(B), check R(B) is array of int */
@@ -290,10 +280,16 @@ OP_RAVI_GETFIELD,     /*	A B C	R(A) := R(B)[RK(C)], string key  */
 OP_RAVI_SELF_SK,      /*	A B C	R(A+1) := R(B); R(A) := R(B)[RK(C)], string key */
 OP_RAVI_SETFIELD,     /*	A B C	R(A)[RK(B)] := RK(C), string key */
 OP_RAVI_GETTABUP_SK,  /*	A B C	R(A) := UpValue[B][RK(C)], string key */
+#ifdef RAVI_DEFER_STATEMENT
 OP_RAVI_DEFER,        /*        A	mark variable A "deferred"	  */
+#endif
 } OpCode;
 
+#ifdef RAVI_DEFER_STATEMENT
 #define NUM_OPCODES	(cast(int, OP_RAVI_DEFER) + 1)
+#else
+#define NUM_OPCODES	(cast(int, OP_RAVI_GETTABUP_SK) + 1)
+#endif
 
 /*===========================================================================
   Notes:

include/lparser.h

@@ -66,15 +66,13 @@ typedef struct expdesc {
       short idx;  /* index (R/K) */
       lu_byte t;  /* table (register or upvalue) */
       lu_byte vt;  /* whether 't' is register (VLOCAL) or upvalue (VUPVAL) */
-      ravi_type_map key_ravi_type_map;  /* Map of possible types the key could have */
-      // lu_byte key_ravi_type; /* RAVI change: key type */
+      lu_byte key_ravi_type; /* RAVI change: key type */
       TString *usertype; /* RAVI change: usertype name */
     } ind;
   } u;
   int t;  /* patch list of 'exit when true' */
   int f;  /* patch list of 'exit when false' */
-  ravi_type_map ravi_type_map;  /* Map of possible types this expression could have */
-  // lu_byte ravi_type; /* RAVI change: type of the expression if known, else RAVI_TANY */
+  lu_byte ravi_type; /* RAVI change: type of the expression if known, else RAVI_TANY */
   TString *usertype; /* RAVI change: usertype name */
   int pc;         /* RAVI change: holds the program counter for OP_NEWTABLE instruction when a constructor expression is parsed */
 } expdesc;
@@ -243,9 +241,7 @@ LUAI_FUNC LClosure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff,
                                  Dyndata *dyd, const char *name, int firstchar);
 
 /** RAVI extensions **/
-#define RAVI_TYPEMAP_MAX_LEN (sizeof("nil|boolean|integer|number|integer[]|number[]|table|string|function|userdata|?|"))
-
-LUAI_FUNC void raviY_typemap_string(ravi_type_map tm, char* buf);
+LUAI_FUNC const char *raviY_typename(ravitype_t tt);
 
 /* Special printf that recognises following conversions:
  * %e - expdesc *
@@ -263,7 +259,7 @@ LUAI_FUNC void raviY_printf(FuncState *fs, const char *format, ...);
  * Else RAVI_TANY is returned. Note that this function only looks
  * at active local variables - see note on FuncState on what this means.
  */
-LUAI_FUNC ravi_type_map raviY_get_register_typeinfo(FuncState *fs, int reg, TString **);
+LUAI_FUNC ravitype_t raviY_get_register_typeinfo(FuncState *fs, int reg, TString **);
 
 #define DEBUG_EXPR(p)                                                          \
   if ((ravi_parser_debug & 1) != 0) {                                          \

include/lstate.h

@@ -15,6 +15,7 @@
 #include "lopcodes.h"
 
 /*
+
 ** Some notes about garbage-collected objects: All objects in Lua must
 ** be kept somehow accessible until being freed, so all objects always
 ** belong to one (and only one) of these lists, using field 'next' of
@@ -26,44 +27,12 @@
 ** 'fixedgc': all objects that are not to be collected (currently
 ** only small strings, such as reserved words).
 **
-** For the generational collector, some of these lists have marks for
-** generations. Each mark points to the first element in the list for
-** that particular generation; that generation goes until the next mark.
-**
-** 'allgc' -> 'survival': new objects;
-** 'survival' -> 'old': objects that survived one collection;
-** 'old1' -> 'reallyold': objects that became old in last collection;
-** 'reallyold' -> NULL: objects old for more than one cycle.
-**
-** 'finobj' -> 'finobjsur': new objects marked for finalization;
-** 'finobjsur' -> 'finobjold1': survived   """";
-** 'finobjold1' -> 'finobjrold': just old  """";
-** 'finobjrold' -> NULL: really old       """".
-**
-** All lists can contain elements older than their main ages, due
-** to 'luaC_checkfinalizer' and 'udata2finalize', which move
-** objects between the normal lists and the "marked for finalization"
-** lists. Moreover, barriers can age young objects in young lists as
-** OLD0, which then become OLD1. However, a list never contains
-** elements younger than their main ages.
-**
-** The generational collector also uses a pointer 'firstold1', which
-** points to the first OLD1 object in the list. It is used to optimize
-** 'markold'. (Potentially OLD1 objects can be anywhere between 'allgc'
-** and 'reallyold', but often the list has no OLD1 objects or they are
-** after 'old1'.) Note the difference between it and 'old1':
-** 'firstold1': no OLD1 objects before this point; there can be all
-**   ages after it.
-** 'old1': no objects younger than OLD1 after this point.
-*/
-
-/*
 ** Moreover, there is another set of lists that control gray objects.
 ** These lists are linked by fields 'gclist'. (All objects that
 ** can become gray have such a field. The field is not the same
 ** in all objects, but it always has this name.)  Any gray object
 ** must belong to one of these lists, and all objects in these lists
-** must be gray (with two exceptions explained below):
+** must be gray:
 **
 ** 'gray': regular gray objects, still waiting to be visited.
 ** 'grayagain': objects that must be revisited at the atomic phase.
@@ -74,17 +43,9 @@
 ** 'weak': tables with weak values to be cleared;
 ** 'ephemeron': ephemeron tables with white->white entries;
 ** 'allweak': tables with weak keys and/or weak values to be cleared.
-**
-** The exceptions to that "gray rule" are:
-** - TOUCHED2 objects in generational mode stay in a gray list (because
-** they must be visited again at the end of the cycle), but they are
-** marked black because assignments to them must activate barriers (to
-** move them back to TOUCHED1).
-** - Open upvales are kept gray to avoid barriers, but they stay out
-** of gray lists. (They don't even have a 'gclist' field.)
-*/
-
+** The last three lists are used only during the atomic phase.
 
+*/
 
 
 struct lua_longjmp;  /* defined in ldo.c */
@@ -106,12 +67,10 @@ struct lua_longjmp;  /* defined in ldo.c */
 
 #define BASIC_STACK_SIZE        (2*LUA_MINSTACK)
 
-#define stacksize(th)	cast_int((th)->stack_last - (th)->stack)
-
 
 /* kinds of Garbage Collection */
-#define KGC_INC		0	/* incremental gc */
-#define KGC_GEN		1	/* generational gc */
+#define KGC_NORMAL	0
+#define KGC_EMERGENCY	1	/* gc was forced by an allocation failure */
 
 
 typedef struct stringtable {
@@ -187,21 +146,15 @@ typedef struct global_State {
   void *ud;         /* auxiliary data to 'frealloc' */
   l_mem totalbytes;  /* number of bytes currently allocated - GCdebt */
   l_mem GCdebt;  /* bytes allocated not yet compensated by the collector */
+  lu_mem GCmemtrav;  /* memory traversed by the GC */
   lu_mem GCestimate;  /* an estimate of the non-garbage memory in use */
-  lu_mem lastatomic;  /* see function 'genstep' in file 'lgc.c' */
   stringtable strt;  /* hash table for strings */
   TValue l_registry;
   unsigned int seed;  /* randomized seed for hashes */
   lu_byte currentwhite;
   lu_byte gcstate;  /* state of garbage collector */
   lu_byte gckind;  /* kind of GC running */
-  lu_byte genminormul;  /* control for minor generational collections */
-  lu_byte genmajormul;  /* control for major generational collections */
   lu_byte gcrunning;  /* true if GC is running */
-  lu_byte gcemergency;  /* true if this is an emergency collection */
-  lu_byte gcpause;  /* size of pause between successive GCs */
-  lu_byte gcstepmul;  /* GC "speed" */
-  lu_byte gcstepsize;  /* (log2 of) GC granularity */
   GCObject *allgc;  /* list of all collectable objects */
   GCObject **sweepgc;  /* current position of sweep in list */
   GCObject *finobj;  /* list of collectable objects with finalizers */
@@ -212,15 +165,10 @@ typedef struct global_State {
   GCObject *allweak;  /* list of all-weak tables */
   GCObject *tobefnz;  /* list of userdata to be GC */
   GCObject *fixedgc;  /* list of objects not to be collected */
-  /* fields for generational collector */
-  GCObject *survival;  /* start of objects that survived one GC cycle */
-  GCObject *old1;  /* start of old1 objects */
-  GCObject *reallyold;  /* objects more than one cycle old ("really old") */
-  GCObject *firstold1;  /* first OLD1 object in the list (if any) */
-  GCObject *finobjsur;  /* list of survival objects with finalizers */
-  GCObject *finobjold1;  /* list of old1 objects with finalizers */
-  GCObject *finobjrold;  /* list of really old objects with finalizers */
   struct lua_State *twups;  /* list of threads with open upvalues */
+  unsigned int gcfinnum;  /* number of finalizers to call in each GC step */
+  int gcpause;  /* size of pause between successive GCs */
+  int gcstepmul;  /* GC 'granularity' */
   lua_CFunction panic;  /* to be called in unprotected errors */
   struct lua_State *mainthread;
   const lua_Number *version;  /* pointer to version number */
@@ -292,7 +240,6 @@ union GCUnion {
   struct Udata u;
   union Closure cl;
   struct Table h;
-  struct RaviArray arr;
   struct Proto p;
   struct lua_State th;  /* thread */
 };
@@ -309,8 +256,7 @@ union GCUnion {
 #define gco2cl(o)  \
 	check_exp(novariant((o)->tt) == LUA_TFUNCTION, &((cast_u(o))->cl))
 /** RAVI change - we have table sub types in RAVI **/
-#define gco2t(o)  check_exp((o)->tt == LUA_TTABLE, &((cast_u(o))->h))
-#define gco2array(o)  check_exp(((o)->tt == RAVI_TIARRAY || (o)->tt == RAVI_TFARRAY), &((cast_u(o))->arr))
+#define gco2t(o)  check_exp(novariant((o)->tt) == LUA_TTABLE, &((cast_u(o))->h))
 #define gco2p(o)  check_exp((o)->tt == LUA_TPROTO, &((cast_u(o))->p))
 #define gco2th(o)  check_exp((o)->tt == LUA_TTHREAD, &((cast_u(o))->th))
 

include/ltable.h

@@ -117,56 +117,56 @@ LUAI_FUNC void luaH_resize (lua_State *L, Table *t, unsigned int nasize,
 LUAI_FUNC void luaH_resizearray (lua_State *L, Table *t, unsigned int nasize);
 LUAI_FUNC void luaH_free (lua_State *L, Table *t);
 LUAI_FUNC int luaH_next (lua_State *L, Table *t, StkId key);
-LUAI_FUNC lua_Unsigned luaH_getn (Table *t);
+LUAI_FUNC int luaH_getn (Table *t);
 
 /* RAVI array specialization */
 
 /* Creates a specialized version of Lua Table to support Ravi's
  * integer[] and number[] arrays.
  */
-LUAI_FUNC RaviArray *raviH_new(lua_State *L, ravi_type_map array_type, int is_slice);
-LUAI_FUNC void raviH_free(lua_State* L, RaviArray* t);
-LUAI_FUNC int raviH_next(lua_State* L, RaviArray* t, StkId key);
+LUAI_FUNC Table *raviH_new(lua_State *L, ravitype_t array_type, int is_slice);
 
-LUAI_FUNC RaviArray *raviH_new_integer_array(lua_State *L, unsigned int len,
+LUAI_FUNC Table *raviH_new_integer_array(lua_State *L, unsigned int len,
                                          lua_Integer init_value);
 
-LUAI_FUNC RaviArray *raviH_new_number_array(lua_State *L, unsigned int len,
+LUAI_FUNC Table *raviH_new_number_array(lua_State *L, unsigned int len,
                                         lua_Number init_value);
 
 /* Returns the array length - note that this function will
  * ignore any elements outside of the Ravi Array structure
  */
-#define raviH_getn(t) ((t)->len - 1)
+LUAI_FUNC int raviH_getn(Table *t);
 
 /* Type specific array set operation */
-LUAI_FUNC void raviH_set_int(lua_State *L, RaviArray *t, lua_Unsigned key,
+LUAI_FUNC void raviH_set_int(lua_State *L, Table *t, lua_Unsigned key,
                              lua_Integer value);
 
 /* Type specific array set operation */
-LUAI_FUNC void raviH_set_float(lua_State *L, RaviArray *t, lua_Unsigned key,
+LUAI_FUNC void raviH_set_float(lua_State *L, Table *t, lua_Unsigned key,
                                lua_Number value);
 
-/* 
-* Create a slice of an existing array
-* The original array is inserted into the
-* the slice as a paren so that
-* the parent array is not garbage collected while this array contains a
-* reference to it. This is enforced in the GC.
+/* Create a slice of an existing array
+* The original table containing the array is inserted into the
+* the slice as a value against special key pointer('key_orig_table') so that
+* the parent table is not garbage collected while this array contains a
+* reference to it
 * The array slice starts at start but start-1 is also accessible because of the
 * implementation having array values starting at 0.
 * A slice must not attempt to release the data array as this is not owned by it,
 * and in fact may point to garbage from a memory allocater's point of view.
 */
-LUAI_FUNC RaviArray *raviH_new_slice(lua_State *L, TValue *parent,
+LUAI_FUNC Table *raviH_new_slice(lua_State *L, TValue *parent,
                                  unsigned int start, unsigned int len);
 
+/* Obtain parent array of the slice */
+LUAI_FUNC const TValue *raviH_slice_parent(lua_State *L, TValue *slice);
+
 /* Type specific array get operation */
 #define raviH_get_int_inline(L, t, key, v)                                     \
   {                                                                            \
     unsigned int ukey = (unsigned int)((key));                                 \
-    lua_Integer *data = (lua_Integer *)t->data;                                \
-    if (ukey < t->len) {                                            \
+    lua_Integer *data = (lua_Integer *)t->ravi_array.data;                     \
+    if (ukey < t->ravi_array.len) {                                            \
       setivalue(v, data[ukey]);                                                \
     } else                                                                     \
       luaG_runerror(L, "array out of bounds");                                 \
@@ -176,8 +176,8 @@ LUAI_FUNC RaviArray *raviH_new_slice(lua_State *L, TValue *parent,
 #define raviH_get_float_inline(L, t, key, v)                                   \
   {                                                                            \
     unsigned int ukey = (unsigned int)((key));                                 \
-    lua_Number *data = (lua_Number *)t->data;                                  \
-    if (ukey < t->len) {                                            \
+    lua_Number *data = (lua_Number *)t->ravi_array.data;                       \
+    if (ukey < t->ravi_array.len) {                                            \
       setfltvalue(v, data[ukey]);                                              \
     } else                                                                     \
       luaG_runerror(L, "array out of bounds");                                 \
@@ -187,8 +187,8 @@ LUAI_FUNC RaviArray *raviH_new_slice(lua_State *L, TValue *parent,
 #define raviH_set_int_inline(L, t, key, value)                                 \
   {                                                                            \
     unsigned int ukey = (unsigned int)((key));                                 \
-    lua_Integer *data = (lua_Integer *)t->data;                                \
-    if (ukey < t->len) {                                            \
+    lua_Integer *data = (lua_Integer *)t->ravi_array.data;                     \
+    if (ukey < t->ravi_array.len) {                                            \
       data[ukey] = (value);                                                    \
     } else                                                                     \
       raviH_set_int(L, t, ukey, (value));                                      \
@@ -198,15 +198,15 @@ LUAI_FUNC RaviArray *raviH_new_slice(lua_State *L, TValue *parent,
 #define raviH_set_float_inline(L, t, key, value)                               \
   {                                                                            \
     unsigned int ukey = (unsigned int)((key));                                 \
-    lua_Number *data = (lua_Number *)t->data;                                  \
-    if (ukey < t->len) {                                            \
+    lua_Number *data = (lua_Number *)t->ravi_array.data;                       \
+    if (ukey < t->ravi_array.len) {                                            \
       data[ukey] = (value);                                                    \
     } else                                                                     \
       raviH_set_float(L, t, ukey, (value));                                    \
   }
 
-LUAI_FUNC void raviH_get_number_array_rawdata(lua_State *L, RaviArray *t, Ravi_NumberArray *data);
-LUAI_FUNC void raviH_get_integer_array_rawdata(lua_State *L, RaviArray *t, Ravi_IntegerArray *data);
+LUAI_FUNC void raviH_get_number_array_rawdata(lua_State *L, Table *t, Ravi_NumberArray *data);
+LUAI_FUNC void raviH_get_integer_array_rawdata(lua_State *L, Table *t, Ravi_IntegerArray *data);
 
 #if defined(LUA_DEBUG)
 LUAI_FUNC Node *luaH_mainposition (const Table *t, const TValue *key);

include/lua.h

@@ -140,10 +140,10 @@ typedef void(*ravi_Writestringerror)(const char *fmt, const char *p);
 #define lua_assert(c)           assert(c)
 
 #if !defined(RAVI_OPTION_STRING1)
-#define RAVI_OPTION_STRING1 "assertions "
+#define RAVI_OPTION_STRING1 " assertions"
 #endif
 
-#define RAVI_OPTION_STRING2 "ltests "
+#define RAVI_OPTION_STRING2 " ltests"
 
 /* to avoid warnings, and to make sure value is really unused */
 #define UNUSED(x)       (x=0, (void)(x))
@@ -164,7 +164,6 @@ typedef struct Memcontrol {
 	unsigned long total;
 	unsigned long maxmem;
 	unsigned long memlimit;
-        unsigned long countlimit;
 	unsigned long objcount[LUA_NUMTAGS];
 } Memcontrol;
 
@@ -226,7 +225,8 @@ extern const char lua_ident[];
 LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud);
 LUA_API void       (lua_close) (lua_State *L);
 LUA_API lua_State *(lua_newthread) (lua_State *L);
-LUA_API int        (lua_resetthread) (lua_State *L);
+
+LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf);
 
 
 LUA_API const lua_Number *(lua_version) (lua_State *L);
@@ -363,14 +363,11 @@ LUA_API int   (lua_pcallk) (lua_State *L, int nargs, int nresults, int errfunc,
                             lua_KContext ctx, lua_KFunction k);
 #define lua_pcall(L,n,r,f)	lua_pcallk(L, (n), (r), (f), 0, NULL)
 
-/* A Lua Closure must be on top of the stack. This will set _ENV upvalue */
-LUA_API void  (ravi_closure_setenv) (lua_State* L);
 LUA_API int   (lua_load) (lua_State *L, lua_Reader reader, void *dt,
                           const char *chunkname, const char *mode);
 
 LUA_API int (lua_dump) (lua_State *L, lua_Writer writer, void *data, int strip);
 
-LUA_API void (raviV_raise_error) (lua_State *L, int errorcode);
 
 /*
 ** coroutine functions
@@ -397,10 +394,8 @@ LUA_API int (lua_isyieldable) (lua_State *L);
 #define LUA_GCSETPAUSE		6
 #define LUA_GCSETSTEPMUL	7
 #define LUA_GCISRUNNING		9
-#define LUA_GCGEN		10
-#define LUA_GCINC		11
 
-LUA_API int (lua_gc) (lua_State *L, int what, ...);
+LUA_API int (lua_gc) (lua_State *L, int what, int data);
 
 
 /*
@@ -639,9 +634,6 @@ LUA_API int ravi_list_code(lua_State *L);
 /* Returns a table with various system limits */
 LUA_API int ravi_get_limits(lua_State *L);
 
-/* Options */
-LUA_API const char *raviV_options(struct lua_State *L);
-
 /* Following are for debugging purposes only */
 LUAI_DDEC int ravi_parser_debug;
 LUA_API void ravi_set_debuglevel(int level);

include/luaconf.h

@@ -61,19 +61,15 @@
 #if defined(LUA_USE_LINUX)
 #define LUA_USE_POSIX
 #define LUA_USE_DLOPEN		/* needs an extra library: -ldl */
-#ifndef LUA_USE_READLINE
 #define LUA_USE_READLINE	/* needs some extra libraries */
 #endif
-#endif
 
 
 #if defined(LUA_USE_MACOSX)
 #define LUA_USE_POSIX
 #define LUA_USE_DLOPEN		/* MacOS does not need -ldl */
-#ifndef LUA_USE_READLINE
 #define LUA_USE_READLINE	/* needs an extra library: -lreadline */
 #endif
-#endif
 
 
 /*
@@ -260,11 +256,9 @@
 #endif				/* } */
 
 
-/*
-** More often than not the libs go together with the core.
-*/
+/* more often than not the libs go together with the core */
 #define LUALIB_API	LUA_API
-#define LUAMOD_API	LUA_API
+#define LUAMOD_API	LUALIB_API
 
 
 /*
@@ -281,17 +275,15 @@
 ** give a warning about it. To avoid these warnings, change to the
 ** default definition.
 */
-#if 0
 #if defined(__GNUC__) && ((__GNUC__*100 + __GNUC_MINOR__) >= 302) && \
     defined(__ELF__)		/* { */
-#define LUAI_FUNC	__attribute__((visibility("internal"))) extern
+/** RAVI change **/
+#define LUAI_FUNC	/* __attribute__((visibility("hidden")))*/ extern
 #else				/* }{ */
 #define LUAI_FUNC	extern
 #endif				/* } */
-#endif
-#define LUAI_FUNC	LUA_API /* AOT code needs to access symbols */
 
-#define LUAI_DDEC	extern
+#define LUAI_DDEC	LUAI_FUNC
 #define LUAI_DDEF	/* empty */
 
 /* }================================================================== */
@@ -429,14 +421,14 @@
 @@ LUA_NUMBER is the floating-point type used by Lua.
 @@ LUAI_UACNUMBER is the result of a 'default argument promotion'
 @@ over a floating number.
-@@ l_floatatt(x) corrects float attribute 'x' to the proper float type
+@@ l_mathlim(x) corrects limit name 'x' to the proper float type
 ** by prefixing it with one of FLT/DBL/LDBL.
 @@ LUA_NUMBER_FRMLEN is the length modifier for writing floats.
 @@ LUA_NUMBER_FMT is the format for writing floats.
 @@ lua_number2str converts a float to a string.
 @@ l_mathop allows the addition of an 'l' or 'f' to all math operations.
 @@ l_floor takes the floor of a float.
-@@ lua_str2number converts a decimal numeral to a number.
+@@ lua_str2number converts a decimal numeric string to a number.
 */
 
 
@@ -448,13 +440,12 @@
 	l_sprintf((s), sz, LUA_NUMBER_FMT, (LUAI_UACNUMBER)(n))
 
 /*
-@@ lua_numbertointeger converts a float number with an integral value
-** to an integer, or returns 0 if float is not within the range of
-** a lua_Integer.  (The range comparisons are tricky because of
-** rounding. The tests here assume a two-complement representation,
-** where MININTEGER always has an exact representation as a float;
-** MAXINTEGER may not have one, and therefore its conversion to float
-** may have an ill-defined value.)
+@@ lua_numbertointeger converts a float number to an integer, or
+** returns 0 if float is not within the range of a lua_Integer.
+** (The range comparisons are tricky because of rounding. The tests
+** here assume a two-complement representation, where MININTEGER always
+** has an exact representation as a float; MAXINTEGER may not have one,
+** and therefore its conversion to float may have an ill-defined value.)
 */
 #define lua_numbertointeger(n,p) \
   ((n) >= (LUA_NUMBER)(LUA_MININTEGER) && \
@@ -468,7 +459,7 @@
 
 #define LUA_NUMBER	float
 
-#define l_floatatt(n)		(FLT_##n)
+#define l_mathlim(n)		(FLT_##n)
 
 #define LUAI_UACNUMBER	double
 
@@ -484,7 +475,7 @@
 
 #define LUA_NUMBER	long double
 
-#define l_floatatt(n)		(LDBL_##n)
+#define l_mathlim(n)		(LDBL_##n)
 
 #define LUAI_UACNUMBER	long double
 
@@ -499,7 +490,7 @@
 
 #define LUA_NUMBER	double
 
-#define l_floatatt(n)		(DBL_##n)
+#define l_mathlim(n)		(DBL_##n)
 
 #define LUAI_UACNUMBER	double
 
@@ -524,13 +515,11 @@
 @@ LUA_UNSIGNED is the unsigned version of LUA_INTEGER.
 **
 @@ LUAI_UACINT is the result of a 'default argument promotion'
-@@ over a LUA_INTEGER.
+@@ over a lUA_INTEGER.
 @@ LUA_INTEGER_FRMLEN is the length modifier for reading/writing integers.
 @@ LUA_INTEGER_FMT is the format for writing integers.
 @@ LUA_MAXINTEGER is the maximum value for a LUA_INTEGER.
 @@ LUA_MININTEGER is the minimum value for a LUA_INTEGER.
-@@ LUA_MAXUNSIGNED is the maximum value for a LUA_UNSIGNED.
-@@ LUA_UNSIGNEDBITS is the number of bits in a LUA_UNSIGNED.
 @@ lua_integer2str converts an integer to a string.
 */
 
@@ -551,9 +540,6 @@
 #define LUA_UNSIGNED		unsigned LUAI_UACINT
 
 
-#define LUA_UNSIGNEDBITS	(sizeof(LUA_UNSIGNED) * CHAR_BIT)
-
-
 /* now the variable definitions */
 
 #if LUA_INT_TYPE == LUA_INT_INT		/* { int */
@@ -564,8 +550,6 @@
 #define LUA_MAXINTEGER		INT_MAX
 #define LUA_MININTEGER		INT_MIN
 
-#define LUA_MAXUNSIGNED		UINT_MAX
-
 #elif LUA_INT_TYPE == LUA_INT_LONG	/* }{ long */
 
 #define LUA_INTEGER		long
@@ -574,8 +558,6 @@
 #define LUA_MAXINTEGER		LONG_MAX
 #define LUA_MININTEGER		LONG_MIN
 
-#define LUA_MAXUNSIGNED		ULONG_MAX
-
 #elif LUA_INT_TYPE == LUA_INT_LONGLONG	/* }{ long long */
 
 /* use presence of macro LLONG_MAX as proxy for C99 compliance */
@@ -588,8 +570,6 @@
 #define LUA_MAXINTEGER		LLONG_MAX
 #define LUA_MININTEGER		LLONG_MIN
 
-#define LUA_MAXUNSIGNED		ULLONG_MAX
-
 #elif defined(LUA_USE_WINDOWS) /* }{ */
 /* in Windows, can use specific Windows types */
 
@@ -599,8 +579,6 @@
 #define LUA_MAXINTEGER		_I64_MAX
 #define LUA_MININTEGER		_I64_MIN
 
-#define LUA_MAXUNSIGNED		_UI64_MAX
-
 #else				/* }{ */
 
 #error "Compiler does not support 'long long'. Use option '-DLUA_32BITS' \
@@ -635,7 +613,7 @@
 
 
 /*
-@@ lua_strx2number converts a hexadecimal numeral to a number.
+@@ lua_strx2number converts an hexadecimal numeric string to a number.
 ** In C99, 'strtod' does that conversion. Otherwise, you can
 ** leave 'lua_strx2number' undefined and Lua will provide its own
 ** implementation.
@@ -653,7 +631,7 @@
 
 
 /*
-@@ lua_number2strx converts a float to a hexadecimal numeral.
+@@ lua_number2strx converts a float to an hexadecimal numeric string.
 ** In C99, 'sprintf' (with format specifiers '%a'/'%A') does that.
 ** Otherwise, you can leave 'lua_number2strx' undefined and Lua will
 ** provide its own implementation.
@@ -699,7 +677,7 @@
 /*
 @@ lua_getlocaledecpoint gets the locale "radix character" (decimal point).
 ** Change that if you do not want to use C locales. (Code using this
-** macro must include the header 'locale.h'.)
+** macro must include header 'locale.h'.)
 */
 #if !defined(lua_getlocaledecpoint)
 #define lua_getlocaledecpoint()		(localeconv()->decimal_point[0])
@@ -740,7 +718,7 @@
 ** {==================================================================
 ** Macros that affect the API and must be stable (that is, must be the
 ** same when you compile Lua and when you compile code that links to
-** Lua).
+** Lua). You probably do not want/need to change them.
 ** =====================================================================
 */
 
@@ -835,7 +813,7 @@
 #endif
 
 
-#if defined(_MSC_VER) && !defined(__clang__)
+#if defined(_MSC_VER)
 
 #define RAVI_NORETURN		__declspec(noreturn)
 #define RAVI_INLINE			__inline
@@ -852,6 +830,12 @@
 /* Following cause the first hash lookup to be inlined,
    and if value is 2 then the second hash lookup is also inlined. */
 #define RAVI_USE_INLINE_SHORTSTR_TGET 1
+#define RAVI_USE_LLVM_BRANCH_WEIGHTS 1
+/* If following is defined as true then LLVM instructions emitted for arithmetic ops 
+   priority floating point ops, else default is to prioritise integer ops */
+#define RAVI_USE_LLVM_ARITH_FLOATPRIORITY 1
+/* Enables the 'defer' statement - RAVI extension */
+#define RAVI_DEFER_STATEMENT
 
 #endif
 

include/lualib.h

@@ -54,10 +54,8 @@ LUAMOD_API int (luaopen_package) (lua_State *L);
 #define LUA_RAVILIBNAME	"ravi"
 LUAMOD_API int (raviopen_jit)(lua_State *L);
 
-#define LUA_RAVICOMPLIBNAME	"compiler"
-LUAMOD_API int (raviopen_compiler)(lua_State *L);
-
-
+#define LUA_ASTLIBNAME	"ast"
+LUAMOD_API int (raviopen_ast_library)(lua_State *L);
 
 /* open all previous libraries */
 LUALIB_API void (luaL_openlibs) (lua_State *L);