issue #151 import LuaJIT's allocator

pull/167/head
Dibyendu Majumdar 6 years ago
commit 73c31bb59d

@ -172,7 +172,7 @@ endif()
set(LUA_CORE_SRCS src/lapi.c src/lcode.c src/lctype.c src/ldebug.c src/ldo.c src/ldump.c
src/lfunc.c src/lgc.c src/llex.c src/lmem.c src/lobject.c src/lopcodes.c
src/lparser.c src/lstate.c src/lstring.c src/ltable.c src/ltm.c src/lundump.c
src/lvm.c src/lzio.c src/ravijit.cpp src/ltests.c src/ravi_profile.c src/ravi_membuf.c
src/lvm.c src/lzio.c src/ravijit.cpp src/ltests.c src/ravi_profile.c src/ravi_ast.c src/ravi_membuf.c
src/ravi_jitshared.c src/bit.c) # src/ravi_alloc.c)
if (COMPUTED_GOTO AND NOT MSVC)
if (CMAKE_C_COMPILER_ID MATCHES "Clang")

@ -310,6 +310,11 @@ LUALIB_API void *raviL_checkudata(lua_State *L, int arg_index, const void *meta_
#define raviL_checkudata(L, arg_index, meta_key) luaL_checkudata(L, arg_index, meta_key)
#endif
LUALIB_API int (raviL_build_ast_from_buffer) (lua_State *L, const char *buff, size_t size,
const char *name, const char *mode);
LUALIB_API int (raviL_dumpast) (lua_State *L);
#endif

@ -77,7 +77,6 @@ int lua_checkmemory (lua_State *L);
/* test for lock/unlock */
struct L_EXTRA { int lock; int *plock; };
#undef LUA_EXTRASPACE
#define LUA_EXTRASPACE sizeof(struct L_EXTRA)
@ -94,7 +93,6 @@ struct L_EXTRA { int lock; int *plock; };
#define lua_unlock(l) lua_assert(--(*getlock(l)->plock) == 0)
LUA_API int luaB_opentests (lua_State *L);
LUA_API void *debug_realloc (void *ud, void *block,

@ -527,6 +527,14 @@ LUA_API void ravi_writestringerror(lua_State *L, const char *fmt, const char *p)
LUA_API void ravi_set_debugger_data(lua_State *L, void *data);
LUA_API void *ravi_get_debugger_data(lua_State *L);
/*
** Experimental (wip) implementation of new
** parser and code generator
*/
LUA_API int (ravi_build_ast_from_buffer) (lua_State *L, lua_Reader reader, void *dt,
const char *chunkname, const char *mode);
/////////////////////////////////////////////////////////////////////////////
/*
Bunch of useful functions for debugging

@ -54,6 +54,9 @@ LUAMOD_API int (luaopen_package) (lua_State *L);
#define LUA_RAVILIBNAME "ravi"
LUAMOD_API int (raviopen_llvmjit)(lua_State *L);
#define LUA_ASTLIBNAME "ast"
LUAMOD_API int (raviopen_ast_library)(lua_State *L);
#ifdef USE_LLVM
#define LUA_LLVMLIBNAME "llvm"
LUAMOD_API int (raviopen_llvmluaapi)(lua_State *L);

@ -0,0 +1,15 @@
/*
** $Id: lparser.h,v 1.74 2014/10/25 11:50:46 roberto Exp $
** Lua Parser
** See Copyright Notice in lua.h
*/
#ifndef ravi_ast_h
#define ravi_ast_h
#include "lparser.h"
#endif

@ -4,6 +4,7 @@
#include <stdbool.h>
#include <stdio.h>
#include <stdint.h>
#include <stdarg.h>
typedef struct {
char *buf;
@ -17,9 +18,11 @@ extern void membuff_resize(membuff_t *mb, size_t new_size);
extern void membuff_free(membuff_t *mb);
extern void membuff_add_string(membuff_t *mb, const char *str);
extern void membuff_add_fstring(membuff_t *mb, const char *str, ...);
extern void membuff_add_vfstring(membuff_t *mb, const char *fmt, va_list args);
extern void membuff_add_bool(membuff_t *mb, bool value);
extern void membuff_add_int(membuff_t *mb, int value);
extern void membuff_add_longlong(membuff_t *mb, int64_t value);
extern void membuff_add_char(membuff_t *mb, char c);
/* strncpy() with guaranteed 0 termination */
extern void ravi_string_copy(char *buf, const char *src, size_t buflen);

@ -0,0 +1,360 @@
-- Gaussian elimination
local assert = assert
local slice, numarray, intarray = table.slice, table.numarray, table.intarray
local write = io.write
local function comp(a: number[], b: number[])
local abs = math.abs
for i = 1, #a do
if abs(a[i] - b[i]) > 1e-10 then
return false
end
end
return true
end
local function copy(a: number[])
local c: number[] = numarray(#a, 0.0)
for i = 1,#a do
c[i] = a[i]
end
return c
end
-- Perform row exchanges as per partial pivoting strategy
-- i.e. swap with the row that has the largest value in the
-- pivot position
-- Updates the nrow[] array
-- i = column
-- n = number of rows
local function partial_pivot_row_exchanges(columns: table, nrow: integer[], i: integer, n: integer)
local p: integer = i
local max: number = 0.0
local a: number[] = @number[]( columns[i] )
local max_set = false
-- find the row from i to n that has
-- max absolute value in column[i]
for row=i, n do
local value: number = a[nrow[row]]
if value < 0.0 then value = -value end
if not max_set then
max = value
max_set = true
p = row
elseif value > max then
p = row
max = value
end
end
if a[p] == 0.0 then
error("no unique solution exists")
end
if nrow[i] ~= nrow[p] then
write('Performing row interchange ', i, ' will be swapped with ', p, "\n")
local temp: integer = nrow[i]
nrow[i] = nrow[p]
nrow[p] = temp
end
end
local function dump_matrix(columns: table, m: integer, n: integer, nrow: integer[])
for i = 1,m do
for j = 1,n do
write(columns[j][nrow[i]], ' ')
end
write("\n")
end
end
local function dump_2darray(s, pivots: number[], n: integer)
write(s, "\n")
for i = 1,n do
for j = 1,n do
write(pivots[(j-1)*n+i], ' ')
end
write("\n")
end
end
local function dump_1darray(s, pivots)
write(s, "\n")
local n: integer = #pivots
for i = 1,n do
write(pivots[i], '\n')
end
end
local function back_substitution(n: integer, columns: table, b: number[], nrow: integer[])
-- Now we do the back substitution
local x: number[] = numarray(n, 0.0)
local a: number[] = @number[]( columns[n] )
write('Performing back substitution\n')
x[n] = b[nrow[n]] / a[nrow[n]]
write('x[', n, '] = b[', n, '] / a[', n, '] = ', x[n], "\n")
for i = n-1,1,-1 do
local sum: number
for j = i+1, n do
a = @number[]( columns[j] )
sum = sum + a[nrow[i]] * x[j]
if j == i+1 then
write('sum = ')
else
write('sum = sum + ')
end
write('a[', i, ', ', j, '] * x[', j, ']', "\n")
end
write('sum = ', sum, '\n')
a = @number[]( columns[i] )
x[i] = (b[nrow[i]] - sum) / a[nrow[i]]
write('x[',i,'] = (b[', i, '] - sum) / a[', i, ', ', i, '] = ', x[i], "\n")
end
return x
end
-- FIXME The U returned is not correct
local function gaussian_solve_LU_decomp(A: number[], b: number[], m: integer, n: integer, omit_row_exchanges)
-- make copies
A = copy(A)
b = copy(b)
assert(m == n)
assert(#b == m)
-- nrow will hold the order of the rows allowing
-- easy interchange of rows
local nrow: integer[] = intarray(n)
local pivots: number[] = numarray(m*n)
for i = 1,n do
-- set diagonals to 1.0
-- pivot[i,i] = 1.0
pivots[(i-1)*n+i] = 1.0
end
-- As ravi matrices are column major we
-- create slices for each column for easy access
-- the vector b can also be treated as an additional
-- column thereby creating the augmented matrix
local columns: table = {}
-- we use i as the row and j a the column
-- first get the column slices
for j = 1,n do
columns[j] = slice(A, (j-1)*m+1, m)
end
columns[n+1] = b
-- initialize the nrow vector
for i = 1,n do
nrow[i] = i
end
for j = 1,n-1 do -- j is the column
if not omit_row_exchanges then
partial_pivot_row_exchanges(columns, nrow, j, m)
end
dump_matrix(columns, n, n+1, nrow)
for i = j+1,m do -- i is the row
-- obtain the column j
local column: number[] = @number[]( columns[j] )
local multiplier: number = column[nrow[i]]/column[nrow[j]]
pivots[(j-1)*n+i] = multiplier
write('m(' .. i .. ',' .. j .. ') = ', column[nrow[i]], ' / ', column[nrow[j]], "\n")
write('Performing R(' .. i .. ') = R(' .. i .. ') - m(' .. i .. ',' .. j .. ') * R(' .. j .. ')\n')
-- For the row i, we need to
-- do row(i) = row(i) - multipler * row(j)
for q = j,n+1 do
local col: number[] = @number[]( columns[q] )
col[nrow[i]] = col[nrow[i]] - multiplier*col[nrow[j]]
end
end
write("Post elimination column ", j, "\n")
dump_matrix(columns, n, n+1, nrow)
end
if columns[n][nrow[n]] == 0.0 then
error("no unique solution exists")
end
return back_substitution(n, columns, b, nrow), pivots, A, nrow
-- Now we do the back substitution
-- local x: number[] = numarray(n, 0.0)
-- local a: number[] = @number[]( columns[n] )
-- write('Performing back substitution\n')
-- x[n] = b[nrow[n]] / a[nrow[n]]
-- write('x[', n, '] = b[', n, '] / a[', n, '] = ', x[n], "\n")
-- for i = n-1,1,-1 do
-- local sum: number
-- for j = i+1, n do
-- a = @number[]( columns[j] )
-- sum = sum + a[nrow[i]] * x[j]
-- if j == i+1 then
-- write('sum = ')
-- else
-- write('sum = sum + ')
-- end
-- write('a[', i, ', ', j, '] * x[', j, ']', "\n")
-- end
-- write('sum = ', sum, '\n')
-- a = @number[]( columns[i] )
-- x[i] = (b[nrow[i]] - sum) / a[nrow[i]]
-- write('x[',i,'] = (b[', i, '] - sum) / a[', i, ', ', i, '] = ', x[i], "\n")
-- end
-- return x
end
local function gaussian_solve(A: number[], b: number[], m: integer, n: integer)
-- make copies
A = copy(A)
b = copy(b)
assert(m == n)
assert(#b == m)
-- nrow will hold the order of the rows allowing
-- easy interchange of rows
local nrow: integer[] = intarray(n)
-- As ravi matrices are column major we
-- create slices for each column for easy access
-- the vector b can also be treated as an additional
-- column thereby creating the augmented matrix
local columns: table = {}
-- we use i as the row and j a the column
-- first get the column slices
for j = 1,n do
columns[j] = slice(A, (j-1)*m+1, m)
end
columns[n+1] = b
-- initialize the nrow vector
for i = 1,n do
nrow[i] = i
end
for j = 1,n-1 do -- j is the column
partial_pivot_row_exchanges(columns, nrow, j, m)
dump_matrix(columns, n, n+1, nrow)
for i = j+1,m do -- i is the row
-- obtain the column j
local column: number[] = @number[]( columns[j] )
local multiplier: number = column[nrow[i]]/column[nrow[j]]
write('m(' .. i .. ',' .. j .. ') = ', column[nrow[i]], ' / ', column[nrow[j]], "\n")
write('Performing R(' .. i .. ') = R(' .. i .. ') - m(' .. i .. ',' .. j .. ') * R(' .. j .. ')\n')
-- For the row i, we need to
-- do row(i) = row(i) - multipler * row(j)
for q = j,n+1 do
local col: number[] = @number[]( columns[q] )
col[nrow[i]] = col[nrow[i]] - multiplier*col[nrow[j]]
end
end
write("Post elimination column ", j, "\n")
dump_matrix(columns, n, n+1, nrow)
end
if columns[n][nrow[n]] == 0.0 then
error("no unique solution exists")
end
return back_substitution(n, columns, b, nrow)
-- Now we do the back substitution
-- local x: number[] = numarray(n, 0.0)
-- local a: number[] = @number[]( columns[n] )
-- write('Performing back substitution\n')
-- x[n] = b[nrow[n]] / a[nrow[n]]
-- write('x[', n, '] = b[', n, '] / a[', n, '] = ', x[n], "\n")
-- for i = n-1,1,-1 do
-- local sum: number
-- for j = i+1, n do
-- a = @number[]( columns[j] )
-- sum = sum + a[nrow[i]] * x[j]
-- if j == i+1 then
-- write('sum = ')
-- else
-- write('sum = sum + ')
-- end
-- write('a[', i, ', ', j, '] * x[', j, ']', "\n")
-- end
-- write('sum = ', sum, '\n')
-- a = @number[]( columns[i] )
-- x[i] = (b[nrow[i]] - sum) / a[nrow[i]]
-- write('x[',i,'] = (b[', i, '] - sum) / a[', i, ', ', i, '] = ', x[i], "\n")
-- end
-- return x
end
local A: number[] = { 4,8,-4; 2,5,1; 1,5,10 }
local b: number[] = { 3,8,5 }
-- control (LAPACK solve)
local expectedx: number[] = { 1,-1,1 }
local x:number[], L: number[], U: number[], rows: integer[] = gaussian_solve_LU_decomp(A, b, 3, 3)
print('expected ', table.unpack(expectedx))
print('got ', table.unpack(x))
assert(comp(x, expectedx))
dump_2darray('L', L, 3)
dump_2darray('U', U, 3)
dump_1darray('rows', rows)
A = @number[] { 2,6,4; 1,-1,3; -1,-9,1 }
b = @number[] { 3,7,5 }
expectedx = @number[] { 0,2,-1 }
x, L, U, rows = gaussian_solve_LU_decomp(A, b, 3, 3)
print('expected ', table.unpack(expectedx))
print('got ', table.unpack(x))
assert(comp(x, expectedx))
dump_2darray('L', L, 3)
dump_2darray('U', U, 3)
dump_1darray('rows', rows)
A = @number[] { 0,1,2,1; 1,1,2,2; 1,2,4,1; 1,1,0,1 }
b = @number[] { 1,-1,-1,2 }
expectedx = @number[] { -1.25, 2.25, -0.75, -0.5 }
x, L, U, rows = gaussian_solve_LU_decomp(A, b, 4, 4)
print('expected ', table.unpack(expectedx))
print('got ', table.unpack(x))
assert(comp(x, expectedx))
dump_2darray('L', L, 4)
dump_2darray('U', U, 4)
dump_1darray('rows', rows)
A = @number[] { 3, -6, 12; -4, 14, -34, -1, 1, 9 }
b = @number[] { 7, -6, -16 }
x, L, U, rows = gaussian_solve_LU_decomp(A, b, 3, 3, true)
print('got ', table.unpack(x))
dump_2darray('L', L, 3)
dump_2darray('U', U, 3)
dump_1darray('rows', rows)
print 'Ok'
--ravi.dumplua(gaussian_solve)
--ravi.dumplua(partial_pivot_row_exchanges)

@ -0,0 +1,69 @@
# Test AST construction
local function dotest(code)
local x = ast.parse(code)
print('--[[' .. code .. '\n]]')
print(x:tostring())
print '--'
end
local x
dotest 'return'
dotest 'return 1'
dotest 'return 42, 4.2, true, "hello"'
dotest 'return a'
dotest 'return 1+2'
dotest 'return 2^3-5*4'
dotest 'return 0.0'
dotest 'return 0'
dotest 'return -0//1'
dotest 'return 3^-1'
dotest 'return (1 + 1)^(50 + 50)'
dotest 'return (-2)^(31 - 2)'
dotest 'return (-3^0 + 5) // 3.0'
dotest 'return -3 % 5'
dotest 'return -((2.0^8 + -(-1)) % 8)/2 * 4 - 3'
dotest 'return -((2^8 + -(-1)) % 8)//2 * 4 - 3'
dotest 'return 0xF0.0 | 0xCC.0 ~ 0xAA & 0xFD'
dotest 'return ~(~0xFF0 | 0xFF0)'
dotest 'return ~~-100024.0'
dotest 'return ((100 << 6) << -4) >> 2'
dotest 'return 2^3^2 == 2^(3^2)'
dotest 'return 2^3*4 == (2^3)*4'
dotest 'return 2.0^-2 == 1/4 and -2^- -2 == - - -4'
dotest 'return not nil and 2 and not(2>3 or 3<2)'
dotest 'return -3-1-5 == 0+0-9'
dotest 'return -2^2 == -4 and (-2)^2 == 4 and 2*2-3-1 == 0'
dotest 'return -3%5 == 2 and -3+5 == 2'
dotest 'return 2*1+3/3 == 3 and 1+2 .. 3*1 == "33"'
dotest 'return not(2+1 > 3*1) and "a".."b" > "a"'
dotest 'return "7" .. 3 << 1 == 146'
dotest 'return 10 >> 1 .. "9" == 0'
dotest 'return 10 | 1 .. "9" == 27'
dotest 'return 0xF0 | 0xCC ~ 0xAA & 0xFD == 0xF4'
dotest 'return 0xFD & 0xAA ~ 0xCC | 0xF0 == 0xF4'
dotest 'return 0xF0 & 0x0F + 1 == 0x10'
dotest 'return 3^4//2^3//5 == 2'
dotest 'return -3+4*5//2^3^2//9+4%10/3 == (-3)+(((4*5)//(2^(3^2)))//9)+((4%10)/3)'
dotest 'return not ((true or false) and nil)'
dotest 'return true or false and nil'
-- old bug
dotest 'return (((1 or false) and true) or false) == true'
dotest 'return (((nil and true) or false) and true) == false'
dotest 'return -(1 or 2) == -1 and (1 and 2)+(-1.25 or -4) == 0.75'
dotest 'return (b or a)+1 == 2 and (10 or a)+1 == 11'
dotest 'return ((2<3) or 1) == true and (2<3 and 4) == 4'
dotest 'return (x>y) and x or y == 2'
dotest 'return (x>y) and x or y == 2'

@ -13,8 +13,8 @@ Quick reminders:
2. lauxlib.c has new functions raviL_loadbufferx() and raviL_dumpast()
3. ldo.c has new functions ravi_f_parser() and raviD_protectedparser()
4. Implementation of raviY_parser() is in ravi_ast.c
5. There is a new Lua api raviload() that calls ravi_load() - this can be used to start developing/testing the new parser.
6. There is a also a test driver test_ast.c.
Its been a while since I started this ... so cannot recollect now what the entry point was. I think the aim was to
allow the new parser to be called by using a new API function. The new parser is a pass through - i.e. it throws away everything
and eventually calls the existing parser so this way I can experiment with it without actually needing to generate code
just yet.
The new parser does nothing yet - it simply ends up calling the standard Lua parser. The idea is that we will have a passthrough
that is only used to test/build the new module without actually interfering with the existing flow.

@ -1332,7 +1332,6 @@ LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data,
return status;
}
LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data, int strip) {
int status;
TValue *o;

@ -760,7 +760,6 @@ LUALIB_API int luaL_loadbufferx (lua_State *L, const char *buff, size_t size,
return lua_load(L, getS, &ls, name, mode);
}
LUALIB_API int luaL_loadstring (lua_State *L, const char *s) {
return luaL_loadbuffer(L, s, strlen(s), s);
}
@ -1070,6 +1069,11 @@ LUALIB_API void luaL_checkversion_ (lua_State *L, lua_Number ver, size_t sz) {
(LUAI_UACNUMBER)ver, (LUAI_UACNUMBER)*v);
}
LUALIB_API int (raviL_dumpast) (lua_State *L) {
(void) L;
return 0;
}
#if 0
/* The normal Lua metatable functions in C use string
keys - these are expensive as the key needs to be

@ -355,6 +355,7 @@ static int luaB_load (lua_State *L) {
return load_aux(L, status, env);
}
/* }====================================================== */

@ -64,6 +64,7 @@ static const luaL_Reg loadedlibs[] = {
{LUA_BITLIBNAME, luaopen_bit32},
#endif
{LUAJIT_BITLIBNAME, luaopen_bit },
{LUA_ASTLIBNAME, raviopen_ast_library},
{NULL, NULL}
};

@ -74,7 +74,7 @@ const char *raviY_typename(ravitype_t tt) {
case RAVI_TNUMFLT:
return "number";
case RAVI_TNUMINT:
return "integr";
return "integer";
case RAVI_TSTRING:
return "string";
case RAVI_TFUNCTION:

File diff suppressed because it is too large Load Diff

@ -58,6 +58,12 @@ void membuff_add_fstring(membuff_t *mb, const char *fmt, ...) {
if (n < 0) { abort(); }
membuff_add_string(mb, buffer);
}
void membuff_add_vfstring(membuff_t *mb, const char *fmt, va_list args) {
char buffer[1024];
int n = vsnprintf(buffer, sizeof buffer, fmt, args);
if (n < 0) { abort(); }
membuff_add_string(mb, buffer);
}
void membuff_add_bool(membuff_t *mb, bool value) {
if (value)
membuff_add_string(mb, "true");
@ -74,3 +80,7 @@ void membuff_add_longlong(membuff_t *mb, int64_t value) {
snprintf(temp, sizeof temp, "%" PRId64 "", value);
membuff_add_string(mb, temp);
}
void membuff_add_char(membuff_t *mb, char c) {
char temp[2] = { c, '\0' };
membuff_add_string(mb, temp);
}

@ -0,0 +1,40 @@
#include <ctype.h>
#include <limits.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <inttypes.h>
#include "lua.h"
#include "lauxlib.h"
#include "lualib.h"
#include "ravi_ast.h"
static int test_buildast(const char *code)
{
int rc = 0;
lua_State *L;
L = luaL_newstate();
if (raviL_build_ast_from_buffer(L, code, strlen(code), "testcode", NULL) != 0) {
rc = 1;
fprintf(stderr, "%s\n", lua_tostring(L, -1));
lua_pop(L, 1); /* pop error message from the stack */
}
else
raviL_dumpast(L);
lua_close(L);
return rc;
}
int main(void)
{
int failures = 0;
//
failures += test_buildast("return");
failures += test_buildast("return 1");
failures += test_buildast("return 1+2");
return failures ? 1 : 0;
}
Loading…
Cancel
Save