e3cc03409e01 — Leonard Ritter a month ago
* Rc: added double free debugging support
* fixed ubsan/asan build option
* disabled extended debug info generation for now, as it can trigger a double free in the LLVM codebase
* small adjustments for building with LLVM 11 (not complete yet)
6 files changed, 141 insertions(+), 32 deletions(-)

M genie.lua
M lib/scopes/Rc.sc
M src/boot.cpp
M src/c_import.cpp
M src/gen_llvm.cpp
M testing/test_rc.sc
M genie.lua +26 -22
@@ 24,6 24,13 @@ end
 local BINDIR = THISDIR .. "/bin"
 
 local USE_ASAN_UBSAN = false
+local ASAN_USAN_OPTS = {
+    "-fsanitize=address",
+    "-fsanitize-address-use-after-scope",
+    "-fno-omit-frame-pointer",
+    "-fsanitize=undefined",
+    "-fno-common",
+}
 
 local function flatten(t)
     local result = {}

          
@@ 112,6 119,10 @@ if not os.is("windows") then
     premake.gcc.cxx = CLANG_CXX
     premake.gcc.cc = CLANG_CC
     premake.gcc.llvm = true
+    if USE_ASAN_UBSAN then
+        premake.gcc.ld = CLANG_CXX
+    end
+
 end
 
 solution "scopes"

          
@@ 261,16 272,9 @@ project "scopesrt"
         }
 
         if USE_ASAN_UBSAN then
-            local opts = {
-                "-fsanitize=address",
-                "-fsanitize-address-use-after-scope",
-                "-fno-omit-frame-pointer",
-                "-fsanitize=undefined",
-                "-fno-common",
-            }
-            buildoptions_cpp(opts)
-            buildoptions_c(opts)
-            linkoptions(opts)
+            buildoptions_cpp(ASAN_USAN_OPTS)
+            buildoptions_c(ASAN_USAN_OPTS)
+            linkoptions(ASAN_USAN_OPTS)
         end
 
         defines {

          
@@ 282,8 286,9 @@ project "scopesrt"
         }
 
         linkoptions {
-            --"-Wl,--stack,8388608"
-            --"-Wl,--stack,16777216"
+            --"-Wl,--stack,8388608",
+            --"-Wl,--stack,16777216",
+            --"-Wl,--stack,67108864", -- 64 GB
             "-Wl,-soname,libscopesrt.so",
             "-Wl,--version-script=" .. THISDIR .. "/src/libscopesrt.map",
         }

          
@@ 422,16 427,9 @@ project "scopesrt"
         }
 
         if USE_ASAN_UBSAN then
-            local opts = {
-                "-fsanitize=address",
-                "-fsanitize-address-use-after-scope",
-                "-fno-omit-frame-pointer",
-                "-fsanitize=undefined",
-                "-fno-common",
-            }
-            buildoptions_cpp(opts)
-            buildoptions_c(opts)
-            linkoptions(opts)
+            buildoptions_cpp(ASAN_USAN_OPTS)
+            buildoptions_c(ASAN_USAN_OPTS)
+            linkoptions(ASAN_USAN_OPTS)
         end
 
         links {

          
@@ 512,6 510,12 @@ project "scopes"
             "-Wl,-rpath=\\$$ORIGIN"
         }
 
+        if USE_ASAN_UBSAN then
+            buildoptions_cpp(ASAN_USAN_OPTS)
+            buildoptions_c(ASAN_USAN_OPTS)
+            linkoptions(ASAN_USAN_OPTS)
+        end
+
         postbuildcommands {
         }
 

          
M lib/scopes/Rc.sc +23 -4
@@ 10,6 10,8 @@ 
     module provides a strong reference type `Rc`, as well as a weak reference
     type `Weak`.
 
+define DEBUG_DOUBLE_FREES false
+
 let
     STRONGRC_INDEX = 0
     WEAKRC_INDEX = 1

          
@@ 35,9 37,26 @@ inline _baseptr (self)
 typedef Weak < ReferenceCounted
 typedef Rc < ReferenceCounted
 
-inline free-rc (self)
-    viewing self
-    free (_baseptr self)
+let use-rc free-rc =
+    static-if DEBUG_DOUBLE_FREES
+        using import Set
+        global used-pointers : (Set (mutable @u8))
+
+        inline use-rc (ptr)
+            'insert used-pointers ptr
+
+        inline free-rc (self)
+            let ptr = (_baseptr self)
+            if ('in? used-pointers ptr)
+                'discard used-pointers ptr
+            else
+                assert false "Rc: double free detected"
+                unreachable;
+            free ptr
+
+        _ use-rc free-rc
+    else
+        _ (inline ()) (inline ())
 
 @@ memo
 inline gen-type (T)

          
@@ 76,6 95,7 @@ inline gen-type (T)
             let mdsize = (sizeof MDT)
             let fullsize = (HEADERSIZE + (sizeof T))
             let ptr = (malloc-array u8 fullsize)
+            use-rc ptr
             let self = (inttoptr (add (ptrtoint ptr usize) HEADERSIZE) storage-type)
             store value self
             let mdptr = (_mdptr self)

          
@@ 176,7 196,6 @@ typedef+ Weak
         store rc refcount
         deref (bitcast (dupe self) RcType)
 
-
 typedef+ Rc
     inline... __typecall
     case (cls, T : type)

          
M src/boot.cpp +1 -0
@@ 40,6 40,7 @@ 
 #endif
 
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/Support/FileSystem.h"
 
 namespace scopes {
 

          
M src/c_import.cpp +8 -5
@@ 322,7 322,7 @@ public:
                 return it->second;
         }
 
-        Symbol name(String::from_stdstring(ed->getName()));
+        Symbol name(String::from_stdstring(ed->getName().data()));
         Symbol typedefname = SYM_Unnamed;
         if (name == SYM_Unnamed) {
             auto tdn = ed->getTypedefNameForAnonDecl();

          
@@ 764,7 764,7 @@ public:
         std::string InternalName = FuncName;
         clang::AsmLabelAttr * asmlabel = f->getAttr<clang::AsmLabelAttr>();
         if(asmlabel) {
-            InternalName = asmlabel->getLabel();
+            InternalName = asmlabel->getLabel().data();
             #ifndef __linux__
                 //In OSX and Windows LLVM mangles assembler labels by adding a '\01' prefix
                 InternalName.insert(InternalName.begin(), '\01');

          
@@ 871,7 871,7 @@ static void add_c_macro(clang::Preproces
         clang::Token tokens[] = { *Tok };
         clang::StringLiteralParser Literal(tokens, PP, false);
         const String *name = String::from_cstr(II->getName().str().c_str());
-        std::string svalue = Literal.GetString();
+        std::string svalue = Literal.GetString().data();
         const String *value = String::from(svalue.c_str(), svalue.size());
         const Anchor *anchor = anchor_from_location(PP.getSourceManager(),
             MI->getDefinitionLoc());

          
@@ 886,13 886,16 @@ static void add_c_macro(clang::Preproces
     clang::SmallString<64> IntegerBuffer;
     bool NumberInvalid = false;
     clang::StringRef Spelling = PP.getSpelling(*Tok, IntegerBuffer, &NumberInvalid);
-    clang::NumericLiteralParser Literal(Spelling, Tok->getLocation(), PP);
+    clang::NumericLiteralParser Literal(Spelling, Tok->getLocation(), PP
+        /*
+        PP.getSourceManager(), PP.getLangOpts(), PP.getTargetInfo(),
+        PP.getDiagnostics()*/);
     if(Literal.hadError)
         return;
     const String *name = String::from_cstr(II->getName().str().c_str());
     std::string suffix;
     if (Literal.hasUDSuffix()) {
-        suffix = Literal.getUDSuffix();
+        suffix = Literal.getUDSuffix().data();
         std::cout << "TODO: macro literal suffix: " << suffix << std::endl;
     }
     const Anchor *anchor = anchor_from_location(PP.getSourceManager(),

          
M src/gen_llvm.cpp +31 -1
@@ 29,6 29,8 @@ 
 #endif
 #include <libgen.h>
 
+#include <deque>
+
 #include <llvm-c/Core.h>
 //#include <llvm-c/ExecutionEngine.h>
 #include <llvm-c/Analysis.h>

          
@@ 61,6 63,8 @@ namespace scopes {
 
 #define SCOPES_LLVM_SUPPORT_DISASSEMBLY 1
 
+#define SCOPES_LLVM_EXTENDED_DEBUG_INFO 0
+
 //------------------------------------------------------------------------------
 // IL->LLVM IR GENERATOR
 //------------------------------------------------------------------------------

          
@@ 318,6 322,7 @@ struct LLVMIRGenerator {
 
     std::unordered_map< PMIntrinsicKey, LLVMValueRef, HashPMIntrinsicKey > pm_intrinsics;
 
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
     LLVMMetadataRef debug_voidT;
     LLVMMetadataRef debug_i1T;
     LLVMMetadataRef debug_i8T;

          
@@ 336,6 341,7 @@ struct LLVMIRGenerator {
     std::vector<const Type *> debug_type_todo;
     std::vector<std::pair<LLVMMetadataRef, LLVMMetadataRef>> debug_type_to_replace;
     LLVMMetadataRef current_debug_block = nullptr;
+#endif
 
     bool use_debug_info = true;
     bool generate_object = false;

          
@@ 1058,6 1064,7 @@ struct LLVMIRGenerator {
         return typeref;
     }
 
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
     SCOPES_RESULT(LLVMMetadataRef) create_llvm_debug_type(const Type *type) {
         SCOPES_RESULT_TYPE(LLVMMetadataRef);
         using namespace llvm::dwarf;

          
@@ 1238,6 1245,7 @@ struct LLVMIRGenerator {
         }
         debug_type_to_replace.clear();
     }
+#endif
 
     static Error *last_llvm_error;
     static void fatal_error_handler(const char *Reason) {

          
@@ 1384,6 1392,7 @@ struct LLVMIRGenerator {
         for (size_t i = 0; i < paramcount; ++i) {
             ParameterRef param = params[i];
             LLVMValueRef val = SCOPES_GET_RESULT(abi_import_argument(param->get_type(), func, k));
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
             if (use_debug_info) {
                 auto subprogram = LLVMGetSubprogram(func);
                 current_debug_block = subprogram;

          
@@ 1399,14 1408,17 @@ struct LLVMIRGenerator {
                 auto expr = LLVMDIBuilderCreateExpression(di_builder, 0, 0);
                 LLVMDIBuilderInsertDeclareAtEnd(di_builder, alloc, divar, expr, anchor_to_location(anchor), LLVMGetInsertBlock(builder));
             }
+#endif
 
             assert(val);
             bind(ValueIndex(param), val);
         }
         SCOPES_CHECK_RESULT(translate_block(node->body));
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
         if (use_debug_info) {
             current_debug_block = nullptr;
         }
+#endif
         return {};
     }
 

          
@@ 1497,6 1509,7 @@ struct LLVMIRGenerator {
 
     SCOPES_RESULT(void) translate_block(const Block &node) {
         SCOPES_RESULT_TYPE(void);
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
         LLVMMetadataRef parent_block = nullptr;
         if (use_debug_info) {
             parent_block = current_debug_block;

          
@@ 1506,15 1519,18 @@ struct LLVMIRGenerator {
                 current_debug_block = LLVMDIBuilderCreateLexicalBlock(di_builder, current_debug_block, difile, anchor->lineno, anchor->column);
             }
         }
+#endif
         for (auto entry : node.body) {
             SCOPES_CHECK_RESULT(translate_instruction(entry));
         }
         if (node.terminator) {
             SCOPES_CHECK_RESULT(translate_instruction(node.terminator));
         }
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
         if (use_debug_info) {
             current_debug_block = parent_block;
         }
+#endif
         return {};
     }
 

          
@@ 1865,6 1881,7 @@ struct LLVMIRGenerator {
         } else {
             val = safe_alloca(ty);
         }
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
         if (use_debug_info) {
             LLVMBasicBlockRef bb = LLVMGetInsertBlock(builder);
             LLVMValueRef func = LLVMGetBasicBlockParent(bb);

          
@@ 1878,6 1895,7 @@ struct LLVMIRGenerator {
             auto expr = LLVMDIBuilderCreateExpression(di_builder, 0, 0);
             LLVMDIBuilderInsertDeclareAtEnd(di_builder, val, divar, expr, anchor_to_location(anchor), LLVMGetInsertBlock(builder));
         }
+#endif
         map_phi({ val }, node);
         return {};
     }

          
@@ 2923,6 2941,7 @@ struct LLVMIRGenerator {
         }
     }
 
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
     void init_debug_types() {
         using namespace llvm::dwarf;
         debug_voidT = LLVMDIBuilderCreateBasicType(di_builder, "void", 4, 0, DW_ATE_unsigned, LLVMDIFlagZero);

          
@@ 2940,6 2959,7 @@ struct LLVMIRGenerator {
         debug_noneT = LLVMDIBuilderCreateBasicType(di_builder, "none", 4, 0, DW_ATE_unsigned, LLVMDIFlagZero); // TODO: Not sure
         debug_rawstringT = LLVMDIBuilderCreatePointerType(di_builder, debug_i8T, PointerType::size() * 8, 0, 0, "", 0);
     }
+#endif
 
     void setup_generate(const char *module_name) {
         module = LLVMModuleCreateWithName(module_name);

          
@@ 2973,11 2993,17 @@ struct LLVMIRGenerator {
                 /*Kind*/ LLVMDWARFEmissionFull,
                 /*DWOId*/ 0,
                 /*SplitDebugInlining*/ true,
-                /*DebugInfoForProfiling*/ false);
+                /*DebugInfoForProfiling*/ false
+                // /*SysRoot*/ "", 0,
+                // /*SDK*/ "", 0
+                );
+
 
             //LLVMAddNamedMetadataOperand(module, "llvm.dbg.cu", dicu);
 
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
             init_debug_types();
+#endif
         }
     }
 

          
@@ 2995,10 3021,12 @@ struct LLVMIRGenerator {
         SCOPES_RESULT_TYPE(void);
         size_t k = SCOPES_GET_RESULT(finalize_types());
         assert(!k);
+#if SCOPES_LLVM_EXTENDED_DEBUG_INFO
         size_t kd = SCOPES_GET_RESULT(finalize_debug_types());
         assert(!kd);
 
         replace_debug_types();
+#endif
 
         LLVMDisposeBuilder(builder);
         LLVMDIBuilderFinalize(di_builder);

          
@@ 3007,6 3035,7 @@ struct LLVMIRGenerator {
 #if SCOPES_DEBUG_CODEGEN
         LLVMDumpModule(module);
 #endif
+        /*
         char *errmsg = NULL;
         if (LLVMVerifyModule(module, LLVMReturnStatusAction, &errmsg)) {
             StyledStream ss(SCOPES_CERR);

          
@@ 3017,6 3046,7 @@ struct LLVMIRGenerator {
             SCOPES_ERROR(CGenBackendFailed, errmsg);
         }
         LLVMDisposeMessage(errmsg);
+        */
         return {};
     }
 

          
M testing/test_rc.sc +52 -0
@@ 324,4 324,56 @@ do
     container = k
     ;
 
+
+# TODO: uncomment and fix
+#do
+    # from https://todo.sr.ht/~duangle/scopes/11
+
+    using import Array
+    using import Rc
+    using import enum
+
+    enum StorageKind
+        Pointer : (Rc this-type)
+        Function : (Rc (Array this-type))
+        TypeReference : Symbol
+
+        inline __copy (self)
+            'apply self
+                inline (T ...)
+                    # uncomment next line to see corruption
+                    # print ...
+                    T
+                        va-map
+                            inline (arg)
+                                copy arg
+                            ...
+
+    fn myfn (n)
+        returning (uniqueof StorageKind -1)
+
+        label done
+            if (n == 0)
+                merge done
+                    do
+                        let newST =
+                            copy
+                                this-function (n + 1)
+                        StorageKind.Pointer (Rc.wrap newST)
+            if (n == 1)
+                merge done
+                    do
+                        local args : (Array StorageKind)
+                        'append args
+                            copy
+                                this-function (n + 1)
+                        # doesn't happen without this return.
+                        return
+                            StorageKind.Function (Rc.wrap (deref args))
+            StorageKind.TypeReference 'unknown
+
+    myfn 0
+    print "done"
+    none
+
 ;
  No newline at end of file